2024-11-12 18:13:46,010 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 2024-11-12 18:13:46,034 main DEBUG Took 0.020110 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 18:13:46,034 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 18:13:46,035 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 18:13:46,036 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 18:13:46,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,077 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 18:13:46,099 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,103 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,103 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,104 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,105 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,107 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,108 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,109 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,110 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,111 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,112 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,113 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 18:13:46,114 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,114 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 18:13:46,116 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 18:13:46,118 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 18:13:46,120 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 18:13:46,121 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 18:13:46,122 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 18:13:46,123 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 18:13:46,134 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 18:13:46,137 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 18:13:46,140 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 18:13:46,140 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 18:13:46,141 main DEBUG createAppenders(={Console}) 2024-11-12 18:13:46,143 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 initialized 2024-11-12 18:13:46,143 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 2024-11-12 18:13:46,143 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@69504ae9 OK. 2024-11-12 18:13:46,144 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 18:13:46,145 main DEBUG OutputStream closed 2024-11-12 18:13:46,145 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 18:13:46,145 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 18:13:46,146 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4c1909a3 OK 2024-11-12 18:13:46,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 18:13:46,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 18:13:46,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 18:13:46,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 18:13:46,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 18:13:46,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 18:13:46,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 18:13:46,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 18:13:46,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 18:13:46,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 18:13:46,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 18:13:46,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 18:13:46,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 18:13:46,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 18:13:46,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 18:13:46,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 18:13:46,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 18:13:46,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 18:13:46,259 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 18:13:46,260 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@bcef303) with optional ClassLoader: null 2024-11-12 18:13:46,260 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 18:13:46,261 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@bcef303] started OK. 2024-11-12T18:13:46,694 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c 2024-11-12 18:13:46,697 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 18:13:46,698 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T18:13:46,720 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-12T18:13:46,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T18:13:46,852 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317, deleteOnExit=true 2024-11-12T18:13:46,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T18:13:46,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/test.cache.data in system properties and HBase conf 2024-11-12T18:13:46,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T18:13:46,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/hadoop.log.dir in system properties and HBase conf 2024-11-12T18:13:46,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T18:13:46,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T18:13:46,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T18:13:47,100 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T18:13:47,333 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T18:13:47,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T18:13:47,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T18:13:47,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T18:13:47,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T18:13:47,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T18:13:47,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T18:13:47,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T18:13:47,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T18:13:47,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T18:13:47,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/nfs.dump.dir in system properties and HBase conf 2024-11-12T18:13:47,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/java.io.tmpdir in system properties and HBase conf 2024-11-12T18:13:47,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T18:13:47,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T18:13:47,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T18:13:48,706 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T18:13:48,806 INFO [Time-limited test {}] log.Log(170): Logging initialized @3996ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T18:13:48,912 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:13:49,010 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T18:13:49,065 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T18:13:49,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T18:13:49,067 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T18:13:49,085 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:13:49,088 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/hadoop.log.dir/,AVAILABLE} 2024-11-12T18:13:49,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T18:13:49,379 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/java.io.tmpdir/jetty-localhost-37875-hadoop-hdfs-3_4_1-tests_jar-_-any-6603297090897583743/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T18:13:49,388 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37875} 2024-11-12T18:13:49,389 INFO [Time-limited test {}] server.Server(415): Started @4579ms 2024-11-12T18:13:49,957 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T18:13:49,976 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T18:13:49,995 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T18:13:49,997 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T18:13:49,997 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T18:13:49,998 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/hadoop.log.dir/,AVAILABLE} 2024-11-12T18:13:49,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T18:13:50,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/java.io.tmpdir/jetty-localhost-46667-hadoop-hdfs-3_4_1-tests_jar-_-any-17578328864458722537/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T18:13:50,146 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:46667} 2024-11-12T18:13:50,147 INFO [Time-limited test {}] server.Server(415): Started @5338ms 2024-11-12T18:13:50,216 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T18:13:50,943 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/data/data1/current/BP-2004937493-172.17.0.3-1731435228362/current, will proceed with Du for space computation calculation, 2024-11-12T18:13:50,946 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/data/data2/current/BP-2004937493-172.17.0.3-1731435228362/current, will proceed with Du for space computation calculation, 2024-11-12T18:13:51,061 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T18:13:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3a48f221338c114 with lease ID 0x4b82846851aa4493: Processing first storage report for DS-5dff2cd5-12c7-4652-b227-deefe1e7531a from datanode DatanodeRegistration(127.0.0.1:39361, datanodeUuid=f7670328-f5c7-49f9-8286-33a4d3629853, infoPort=36577, infoSecurePort=0, ipcPort=40407, storageInfo=lv=-57;cid=testClusterID;nsid=1191934172;c=1731435228362) 2024-11-12T18:13:51,144 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3a48f221338c114 with lease ID 0x4b82846851aa4493: from storage DS-5dff2cd5-12c7-4652-b227-deefe1e7531a node DatanodeRegistration(127.0.0.1:39361, datanodeUuid=f7670328-f5c7-49f9-8286-33a4d3629853, infoPort=36577, infoSecurePort=0, ipcPort=40407, storageInfo=lv=-57;cid=testClusterID;nsid=1191934172;c=1731435228362), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T18:13:51,145 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3a48f221338c114 with lease ID 0x4b82846851aa4493: Processing first storage report for DS-a5922689-94f0-481b-8cfe-f1f29f65815b from datanode DatanodeRegistration(127.0.0.1:39361, datanodeUuid=f7670328-f5c7-49f9-8286-33a4d3629853, infoPort=36577, infoSecurePort=0, ipcPort=40407, storageInfo=lv=-57;cid=testClusterID;nsid=1191934172;c=1731435228362) 2024-11-12T18:13:51,145 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3a48f221338c114 with lease ID 0x4b82846851aa4493: from storage DS-a5922689-94f0-481b-8cfe-f1f29f65815b node DatanodeRegistration(127.0.0.1:39361, datanodeUuid=f7670328-f5c7-49f9-8286-33a4d3629853, infoPort=36577, infoSecurePort=0, ipcPort=40407, storageInfo=lv=-57;cid=testClusterID;nsid=1191934172;c=1731435228362), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T18:13:51,246 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c 2024-11-12T18:13:51,373 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/zookeeper_0, clientPort=59986, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T18:13:51,387 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59986 2024-11-12T18:13:51,408 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:51,412 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:51,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741825_1001 (size=7) 2024-11-12T18:13:52,203 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 with version=8 2024-11-12T18:13:52,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/hbase-staging 2024-11-12T18:13:52,302 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T18:13:52,584 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9911683f163c:0 server-side Connection retries=45 2024-11-12T18:13:52,601 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:13:52,602 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T18:13:52,608 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T18:13:52,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:13:52,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T18:13:52,795 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T18:13:52,877 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T18:13:52,889 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T18:13:52,895 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T18:13:52,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1021 (auto-detected) 2024-11-12T18:13:52,936 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-12T18:13:52,962 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39807 2024-11-12T18:13:52,985 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39807 connecting to ZooKeeper ensemble=127.0.0.1:59986 2024-11-12T18:13:53,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398070x0, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T18:13:53,049 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39807-0x100353303020000 connected 2024-11-12T18:13:53,133 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:53,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:53,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:13:53,156 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7, hbase.cluster.distributed=false 2024-11-12T18:13:53,185 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T18:13:53,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39807 2024-11-12T18:13:53,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39807 2024-11-12T18:13:53,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39807 2024-11-12T18:13:53,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39807 2024-11-12T18:13:53,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39807 2024-11-12T18:13:53,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9911683f163c:0 server-side Connection retries=45 2024-11-12T18:13:53,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:13:53,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T18:13:53,372 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T18:13:53,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T18:13:53,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T18:13:53,377 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T18:13:53,380 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T18:13:53,382 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43757 2024-11-12T18:13:53,385 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43757 connecting to ZooKeeper ensemble=127.0.0.1:59986 2024-11-12T18:13:53,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:53,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:53,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437570x0, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T18:13:53,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:437570x0, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:13:53,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43757-0x100353303020001 connected 2024-11-12T18:13:53,418 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T18:13:53,432 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T18:13:53,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T18:13:53,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T18:13:53,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43757 2024-11-12T18:13:53,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43757 2024-11-12T18:13:53,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43757 2024-11-12T18:13:53,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43757 2024-11-12T18:13:53,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43757 2024-11-12T18:13:53,492 DEBUG [M:0;9911683f163c:39807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9911683f163c:39807 2024-11-12T18:13:53,493 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9911683f163c,39807,1731435232358 2024-11-12T18:13:53,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:13:53,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:13:53,509 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9911683f163c,39807,1731435232358 2024-11-12T18:13:53,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T18:13:53,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:53,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:53,573 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T18:13:53,575 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9911683f163c,39807,1731435232358 from backup master directory 2024-11-12T18:13:53,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:13:53,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9911683f163c,39807,1731435232358 2024-11-12T18:13:53,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T18:13:53,590 WARN [master/9911683f163c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T18:13:53,590 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9911683f163c,39807,1731435232358 2024-11-12T18:13:53,593 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T18:13:53,596 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T18:13:53,686 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/hbase.id] with ID: 0bcba757-a082-44ff-81f8-05d913183c06 2024-11-12T18:13:53,686 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/.tmp/hbase.id 2024-11-12T18:13:53,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741826_1002 (size=42) 2024-11-12T18:13:53,708 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/.tmp/hbase.id]:[hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/hbase.id] 2024-11-12T18:13:53,773 INFO [master/9911683f163c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T18:13:53,779 INFO [master/9911683f163c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T18:13:53,811 INFO [master/9911683f163c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 29ms. 2024-11-12T18:13:53,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:53,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:53,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741827_1003 (size=196) 2024-11-12T18:13:53,901 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:13:53,906 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T18:13:53,929 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:53,934 INFO [master/9911683f163c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:13:54,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741828_1004 (size=1189) 2024-11-12T18:13:54,450 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store 2024-11-12T18:13:54,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741829_1005 (size=34) 2024-11-12T18:13:54,495 INFO [master/9911683f163c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T18:13:54,502 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:13:54,504 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T18:13:54,504 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:13:54,505 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:13:54,506 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T18:13:54,506 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:13:54,507 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:13:54,508 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731435234504Disabling compacts and flushes for region at 1731435234504Disabling writes for close at 1731435234506 (+2 ms)Writing region close event to WAL at 1731435234507 (+1 ms)Closed at 1731435234507 2024-11-12T18:13:54,511 WARN [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/.initializing 2024-11-12T18:13:54,511 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/WALs/9911683f163c,39807,1731435232358 2024-11-12T18:13:54,522 INFO [master/9911683f163c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T18:13:54,548 INFO [master/9911683f163c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C39807%2C1731435232358, suffix=, logDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/WALs/9911683f163c,39807,1731435232358, archiveDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/oldWALs, maxLogs=10 2024-11-12T18:13:54,590 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/WALs/9911683f163c,39807,1731435232358/9911683f163c%2C39807%2C1731435232358.1731435234555, exclude list is [], retry=0 2024-11-12T18:13:54,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39361,DS-5dff2cd5-12c7-4652-b227-deefe1e7531a,DISK] 2024-11-12T18:13:54,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T18:13:54,688 INFO [master/9911683f163c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/WALs/9911683f163c,39807,1731435232358/9911683f163c%2C39807%2C1731435232358.1731435234555 2024-11-12T18:13:54,689 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577)] 2024-11-12T18:13:54,690 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:13:54,691 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:13:54,696 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,697 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T18:13:54,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:54,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:54,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T18:13:54,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:54,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:13:54,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T18:13:54,834 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:54,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:13:54,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T18:13:54,839 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:54,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:13:54,841 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,845 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,846 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,852 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,853 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,858 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T18:13:54,863 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T18:13:54,867 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:13:54,869 INFO [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59531476, jitterRate=-0.1129118800163269}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T18:13:54,877 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731435234715Initializing all the Stores at 1731435234719 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435234720 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435234721 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435234721Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435234721Cleaning up temporary data from old regions at 1731435234853 (+132 ms)Region opened successfully at 1731435234877 (+24 ms) 2024-11-12T18:13:54,881 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T18:13:54,961 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c88a7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9911683f163c/172.17.0.3:0 2024-11-12T18:13:55,031 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T18:13:55,051 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T18:13:55,052 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T18:13:55,056 INFO [master/9911683f163c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T18:13:55,060 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-12T18:13:55,066 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T18:13:55,067 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T18:13:55,133 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T18:13:55,146 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T18:13:55,151 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T18:13:55,155 INFO [master/9911683f163c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T18:13:55,157 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T18:13:55,159 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T18:13:55,167 INFO [master/9911683f163c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T18:13:55,172 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T18:13:55,176 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T18:13:55,184 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T18:13:55,188 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T18:13:55,220 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T18:13:55,225 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T18:13:55,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T18:13:55,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:55,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T18:13:55,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:55,252 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9911683f163c,39807,1731435232358, sessionid=0x100353303020000, setting cluster-up flag (Was=false) 2024-11-12T18:13:55,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:55,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:55,305 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T18:13:55,311 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9911683f163c,39807,1731435232358 2024-11-12T18:13:55,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:55,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:55,338 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T18:13:55,340 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9911683f163c,39807,1731435232358 2024-11-12T18:13:55,350 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T18:13:55,397 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(746): ClusterId : 0bcba757-a082-44ff-81f8-05d913183c06 2024-11-12T18:13:55,401 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T18:13:55,408 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T18:13:55,409 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T18:13:55,414 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T18:13:55,415 DEBUG [RS:0;9911683f163c:43757 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5429df53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9911683f163c/172.17.0.3:0 2024-11-12T18:13:55,441 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9911683f163c:43757 2024-11-12T18:13:55,444 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T18:13:55,446 INFO [RS:0;9911683f163c:43757 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T18:13:55,446 INFO [RS:0;9911683f163c:43757 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T18:13:55,446 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T18:13:55,450 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(2659): reportForDuty to master=9911683f163c,39807,1731435232358 with port=43757, startcode=1731435233300 2024-11-12T18:13:55,459 INFO [master/9911683f163c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T18:13:55,468 DEBUG [RS:0;9911683f163c:43757 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T18:13:55,469 INFO [master/9911683f163c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T18:13:55,477 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9911683f163c,39807,1731435232358 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T18:13:55,491 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:13:55,492 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:13:55,493 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:13:55,493 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9911683f163c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T18:13:55,493 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9911683f163c:0, corePoolSize=10, maxPoolSize=10 2024-11-12T18:13:55,494 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,495 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9911683f163c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T18:13:55,495 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,537 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T18:13:55,538 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T18:13:55,539 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731435265538 2024-11-12T18:13:55,540 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T18:13:55,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50633, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T18:13:55,544 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T18:13:55,549 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T18:13:55,550 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T18:13:55,550 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T18:13:55,550 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T18:13:55,551 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,554 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:55,554 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T18:13:55,554 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:55,558 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T18:13:55,560 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T18:13:55,560 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T18:13:55,564 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T18:13:55,564 INFO [master/9911683f163c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T18:13:55,569 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.large.0-1731435235567,5,FailOnTimeoutGroup] 2024-11-12T18:13:55,571 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.small.0-1731435235569,5,FailOnTimeoutGroup] 2024-11-12T18:13:55,572 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,572 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T18:13:55,573 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,573 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741831_1007 (size=1321) 2024-11-12T18:13:55,590 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T18:13:55,591 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:13:55,598 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T18:13:55,598 WARN [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T18:13:55,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741832_1008 (size=32) 2024-11-12T18:13:55,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:13:55,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T18:13:55,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T18:13:55,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:55,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:55,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T18:13:55,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T18:13:55,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:55,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:55,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T18:13:55,636 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T18:13:55,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:55,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:55,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T18:13:55,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T18:13:55,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:55,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:55,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T18:13:55,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740 2024-11-12T18:13:55,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740 2024-11-12T18:13:55,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T18:13:55,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T18:13:55,659 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:13:55,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T18:13:55,672 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:13:55,677 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61474565, jitterRate=-0.08395759761333466}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:13:55,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731435235611Initializing all the Stores at 1731435235613 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435235613Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435235614 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435235616 (+2 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435235616Cleaning up temporary data from old regions at 1731435235657 (+41 ms)Region opened successfully at 1731435235682 (+25 ms) 2024-11-12T18:13:55,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T18:13:55,682 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T18:13:55,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T18:13:55,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T18:13:55,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T18:13:55,693 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T18:13:55,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731435235682Disabling compacts and flushes for region at 1731435235682Disabling writes for close at 1731435235685 (+3 ms)Writing region close event to WAL at 1731435235693 (+8 ms)Closed at 1731435235693 2024-11-12T18:13:55,700 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(2659): reportForDuty to master=9911683f163c,39807,1731435232358 with port=43757, startcode=1731435233300 2024-11-12T18:13:55,705 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9911683f163c,43757,1731435233300 2024-11-12T18:13:55,709 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] master.ServerManager(517): Registering regionserver=9911683f163c,43757,1731435233300 2024-11-12T18:13:55,729 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T18:13:55,729 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T18:13:55,734 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:13:55,735 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41781 2024-11-12T18:13:55,735 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T18:13:55,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T18:13:55,762 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:13:55,770 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T18:13:55,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T18:13:55,816 DEBUG [RS:0;9911683f163c:43757 {}] zookeeper.ZKUtil(111): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9911683f163c,43757,1731435233300 2024-11-12T18:13:55,817 WARN [RS:0;9911683f163c:43757 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T18:13:55,817 INFO [RS:0;9911683f163c:43757 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:13:55,817 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300 2024-11-12T18:13:55,822 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9911683f163c,43757,1731435233300] 2024-11-12T18:13:55,858 INFO [RS:0;9911683f163c:43757 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T18:13:55,882 INFO [RS:0;9911683f163c:43757 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T18:13:55,893 INFO [RS:0;9911683f163c:43757 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T18:13:55,893 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,895 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T18:13:55,908 INFO [RS:0;9911683f163c:43757 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T18:13:55,910 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,911 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,911 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,911 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,911 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,912 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,912 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9911683f163c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T18:13:55,912 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,912 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,913 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,913 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,913 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,917 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9911683f163c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T18:13:55,917 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:13:55,917 DEBUG [RS:0;9911683f163c:43757 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T18:13:55,921 WARN [9911683f163c:39807 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-12T18:13:55,924 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,925 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,925 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,925 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,925 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,925 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,43757,1731435233300-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T18:13:55,965 INFO [RS:0;9911683f163c:43757 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T18:13:55,968 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,43757,1731435233300-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,969 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:55,969 INFO [RS:0;9911683f163c:43757 {}] regionserver.Replication(171): 9911683f163c,43757,1731435233300 started 2024-11-12T18:13:56,023 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:56,024 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1482): Serving as 9911683f163c,43757,1731435233300, RpcServer on 9911683f163c/172.17.0.3:43757, sessionid=0x100353303020001 2024-11-12T18:13:56,026 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T18:13:56,026 DEBUG [RS:0;9911683f163c:43757 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9911683f163c,43757,1731435233300 2024-11-12T18:13:56,026 DEBUG [RS:0;9911683f163c:43757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,43757,1731435233300' 2024-11-12T18:13:56,029 DEBUG [RS:0;9911683f163c:43757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T18:13:56,030 DEBUG [RS:0;9911683f163c:43757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T18:13:56,031 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T18:13:56,031 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T18:13:56,031 DEBUG [RS:0;9911683f163c:43757 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9911683f163c,43757,1731435233300 2024-11-12T18:13:56,032 DEBUG [RS:0;9911683f163c:43757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9911683f163c,43757,1731435233300' 2024-11-12T18:13:56,032 DEBUG [RS:0;9911683f163c:43757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T18:13:56,037 DEBUG [RS:0;9911683f163c:43757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T18:13:56,041 DEBUG [RS:0;9911683f163c:43757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T18:13:56,041 INFO [RS:0;9911683f163c:43757 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T18:13:56,042 INFO [RS:0;9911683f163c:43757 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T18:13:56,152 INFO [RS:0;9911683f163c:43757 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T18:13:56,161 INFO [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C43757%2C1731435233300, suffix=, logDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300, archiveDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/oldWALs, maxLogs=32 2024-11-12T18:13:56,187 DEBUG [RS:0;9911683f163c:43757 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300/9911683f163c%2C43757%2C1731435233300.1731435236165, exclude list is [], retry=0 2024-11-12T18:13:56,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39361,DS-5dff2cd5-12c7-4652-b227-deefe1e7531a,DISK] 2024-11-12T18:13:56,213 INFO [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300/9911683f163c%2C43757%2C1731435233300.1731435236165 2024-11-12T18:13:56,214 DEBUG [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577)] 2024-11-12T18:13:56,424 DEBUG [9911683f163c:39807 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T18:13:56,441 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:13:56,449 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,43757,1731435233300, state=OPENING 2024-11-12T18:13:56,491 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T18:13:56,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:56,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:13:56,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:13:56,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:13:56,533 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T18:13:56,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:13:56,722 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T18:13:56,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52557, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T18:13:56,741 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T18:13:56,742 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T18:13:56,743 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T18:13:56,747 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9911683f163c%2C43757%2C1731435233300.meta, suffix=.meta, logDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300, archiveDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/oldWALs, maxLogs=32 2024-11-12T18:13:56,773 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300/9911683f163c%2C43757%2C1731435233300.meta.1731435236750.meta, exclude list is [], retry=0 2024-11-12T18:13:56,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39361,DS-5dff2cd5-12c7-4652-b227-deefe1e7531a,DISK] 2024-11-12T18:13:56,783 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/WALs/9911683f163c,43757,1731435233300/9911683f163c%2C43757%2C1731435233300.meta.1731435236750.meta 2024-11-12T18:13:56,784 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36577:36577)] 2024-11-12T18:13:56,785 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:13:56,787 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T18:13:56,790 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T18:13:56,796 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T18:13:56,802 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T18:13:56,803 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:13:56,803 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T18:13:56,803 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T18:13:56,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T18:13:56,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T18:13:56,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:56,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:56,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T18:13:56,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T18:13:56,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:56,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:56,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T18:13:56,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T18:13:56,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:56,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:56,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T18:13:56,818 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T18:13:56,818 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:56,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T18:13:56,819 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T18:13:56,821 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740 2024-11-12T18:13:56,824 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740 2024-11-12T18:13:56,826 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T18:13:56,827 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T18:13:56,828 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:13:56,830 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T18:13:56,832 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67664822, jitterRate=0.008284419775009155}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:13:56,832 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T18:13:56,834 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731435236804Writing region info on filesystem at 1731435236804Initializing all the Stores at 1731435236807 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435236807Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435236807Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435236807Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731435236807Cleaning up temporary data from old regions at 1731435236827 (+20 ms)Running coprocessor post-open hooks at 1731435236832 (+5 ms)Region opened successfully at 1731435236834 (+2 ms) 2024-11-12T18:13:56,862 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731435236709 2024-11-12T18:13:56,877 DEBUG [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T18:13:56,878 INFO [RS_OPEN_META-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T18:13:56,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:13:56,883 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9911683f163c,43757,1731435233300, state=OPEN 2024-11-12T18:13:57,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:13:57,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T18:13:57,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:13:57,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T18:13:57,041 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:57,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T18:13:57,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9911683f163c,43757,1731435233300 in 506 msec 2024-11-12T18:13:57,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T18:13:57,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.3030 sec 2024-11-12T18:13:57,060 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T18:13:57,060 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T18:13:57,086 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:57,088 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:57,115 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:57,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57993, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:57,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7520 sec 2024-11-12T18:13:57,141 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731435237140, completionTime=-1 2024-11-12T18:13:57,143 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T18:13:57,144 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T18:13:57,180 INFO [master/9911683f163c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-12T18:13:57,181 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731435297180 2024-11-12T18:13:57,181 INFO [master/9911683f163c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731435357181 2024-11-12T18:13:57,181 INFO [master/9911683f163c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 37 msec 2024-11-12T18:13:57,183 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,39807,1731435232358-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,183 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,39807,1731435232358-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,184 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,39807,1731435232358-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,185 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9911683f163c:39807, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,186 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,186 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,193 DEBUG [master/9911683f163c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T18:13:57,218 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.628sec 2024-11-12T18:13:57,220 INFO [master/9911683f163c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T18:13:57,221 INFO [master/9911683f163c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T18:13:57,222 INFO [master/9911683f163c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T18:13:57,223 INFO [master/9911683f163c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T18:13:57,223 INFO [master/9911683f163c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T18:13:57,224 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,39807,1731435232358-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T18:13:57,225 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,39807,1731435232358-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T18:13:57,234 DEBUG [master/9911683f163c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T18:13:57,236 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T18:13:57,236 INFO [master/9911683f163c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9911683f163c,39807,1731435232358-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T18:13:57,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:57,313 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T18:13:57,313 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T18:13:57,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:57,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:57,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:57,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db9d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:57,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:57,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:57,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:57,347 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:57,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:57,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9911683f163c,39807,1731435232358 2024-11-12T18:13:57,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=703, ProcessCount=11, AvailableMemoryMB=3283 2024-11-12T18:13:57,413 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:13:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:13:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-11-12T18:13:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:13:57,426 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:13:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:13:57,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:13:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T18:13:57,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:13:57,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T18:13:57,450 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:57,454 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:13:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:13:57,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741835_1011 (size=1061) 2024-11-12T18:13:57,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:13:57,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741836_1012 (size=53) 2024-11-12T18:13:57,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:13:57,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 64a54f5030b48a2f7caee1e78dcb2577, disabling compactions & flushes 2024-11-12T18:13:57,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. after waiting 0 ms 2024-11-12T18:13:57,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64a54f5030b48a2f7caee1e78dcb2577: Waiting for close lock at 1731435237510Disabling compacts and flushes for region at 1731435237510Disabling writes for close at 1731435237510Writing region close event to WAL at 1731435237510Closed at 1731435237510 2024-11-12T18:13:57,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:13:57,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731435237514"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435237514"}]},"ts":"1731435237514"} 2024-11-12T18:13:57,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T18:13:57,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:13:57,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435237528"}]},"ts":"1731435237528"} 2024-11-12T18:13:57,536 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T18:13:57,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, ASSIGN}] 2024-11-12T18:13:57,541 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, ASSIGN 2024-11-12T18:13:57,543 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, ASSIGN; state=OFFLINE, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=false 2024-11-12T18:13:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:13:57,696 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=64a54f5030b48a2f7caee1e78dcb2577, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:13:57,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, ASSIGN because future has completed 2024-11-12T18:13:57,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:13:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:13:57,869 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,869 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:13:57,870 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,870 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:13:57,870 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,870 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,874 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,879 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:13:57,880 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64a54f5030b48a2f7caee1e78dcb2577 columnFamilyName A 2024-11-12T18:13:57,880 DEBUG [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:57,881 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.HStore(327): Store=64a54f5030b48a2f7caee1e78dcb2577/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:13:57,882 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,885 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:13:57,886 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64a54f5030b48a2f7caee1e78dcb2577 columnFamilyName B 2024-11-12T18:13:57,886 DEBUG [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:57,888 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.HStore(327): Store=64a54f5030b48a2f7caee1e78dcb2577/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:13:57,888 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,891 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:13:57,892 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64a54f5030b48a2f7caee1e78dcb2577 columnFamilyName C 2024-11-12T18:13:57,892 DEBUG [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:13:57,894 INFO [StoreOpener-64a54f5030b48a2f7caee1e78dcb2577-1 {}] regionserver.HStore(327): Store=64a54f5030b48a2f7caee1e78dcb2577/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:13:57,894 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,895 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,897 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,898 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,901 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,902 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,903 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:13:57,907 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,917 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:13:57,920 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 64a54f5030b48a2f7caee1e78dcb2577; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74379454, jitterRate=0.10834023356437683}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:13:57,921 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:57,923 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 64a54f5030b48a2f7caee1e78dcb2577: Running coprocessor pre-open hook at 1731435237871Writing region info on filesystem at 1731435237871Initializing all the Stores at 1731435237873 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435237873Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435237873Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435237873Cleaning up temporary data from old regions at 1731435237902 (+29 ms)Running coprocessor post-open hooks at 1731435237921 (+19 ms)Region opened successfully at 1731435237922 (+1 ms) 2024-11-12T18:13:57,926 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., pid=6, masterSystemTime=1731435237860 2024-11-12T18:13:57,934 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=64a54f5030b48a2f7caee1e78dcb2577, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:13:57,936 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,936 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:57,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:13:57,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T18:13:57,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 in 239 msec 2024-11-12T18:13:57,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T18:13:57,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, ASSIGN in 410 msec 2024-11-12T18:13:57,954 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:13:57,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435237954"}]},"ts":"1731435237954"} 2024-11-12T18:13:57,958 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T18:13:57,960 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:13:57,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 523 msec 2024-11-12T18:13:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T18:13:58,092 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:13:58,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,100 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7983b71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,105 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e34b7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,107 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,115 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d5e92ab 2024-11-12T18:13:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:13:58,123 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:13:58,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,170 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,175 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,189 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a081598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,196 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a6bf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,199 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,204 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2f81a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,211 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,214 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545d818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c710cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,220 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:13:58,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:13:58,224 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:13:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:13:58,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:13:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d56454e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:13:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:13:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:13:58,226 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:13:58,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6514b0c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:13:58,232 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,232 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,232 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,232 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,234 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,241 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:13:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,257 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T18:13:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,261 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:13:58,261 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:13:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,265 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:13:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:13:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,271 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:13:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:13:58,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:13:58,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:13:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:13:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:58,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:13:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:13:58,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:13:58,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:13:58,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:13:58,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:13:58,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:13:58,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:13:58,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:58,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:58,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7d0fee488e794bfb8105cc765c3404a2 is 50, key is test_row_0/A:col10/1731435238318/Put/seqid=0 2024-11-12T18:13:58,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741837_1013 (size=12001) 2024-11-12T18:13:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:13:58,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7d0fee488e794bfb8105cc765c3404a2 2024-11-12T18:13:58,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435298546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435238478, max timestamp=1731435238561Num uniques 859; 2024-11-12T18:13:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435238478, max timestamp=1731435238561Num uniques 859; 2024-11-12T18:13:58,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:13:58,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:13:58,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435238478, max timestamp=1731435238561Num uniques 859; 2024-11-12T18:13:58,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:13:58,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:58,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e4b4de17ffa54271a55849e7f626bb22 is 50, key is test_row_0/B:col10/1731435238318/Put/seqid=0 2024-11-12T18:13:58,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435298755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435298757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435298759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435298772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435298773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741838_1014 (size=12001) 2024-11-12T18:13:58,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e4b4de17ffa54271a55849e7f626bb22 2024-11-12T18:13:58,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/aa3e1b84d5e649f8b51ec88715692af3 is 50, key is test_row_0/C:col10/1731435238318/Put/seqid=0 2024-11-12T18:13:58,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741839_1015 (size=12001) 2024-11-12T18:13:58,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/aa3e1b84d5e649f8b51ec88715692af3 2024-11-12T18:13:58,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7d0fee488e794bfb8105cc765c3404a2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7d0fee488e794bfb8105cc765c3404a2 2024-11-12T18:13:58,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435298881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435298881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:13:58,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:58,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435298882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435298891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435298901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7d0fee488e794bfb8105cc765c3404a2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T18:13:59,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e4b4de17ffa54271a55849e7f626bb22 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e4b4de17ffa54271a55849e7f626bb22 2024-11-12T18:13:59,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e4b4de17ffa54271a55849e7f626bb22, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T18:13:59,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/aa3e1b84d5e649f8b51ec88715692af3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/aa3e1b84d5e649f8b51ec88715692af3 2024-11-12T18:13:59,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/aa3e1b84d5e649f8b51ec88715692af3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T18:13:59,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 64a54f5030b48a2f7caee1e78dcb2577 in 760ms, sequenceid=12, compaction requested=false 2024-11-12T18:13:59,113 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T18:13:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:13:59,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:59,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T18:13:59,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:13:59,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:13:59,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:13:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:59,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:13:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:13:59,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:13:59,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/14775c1f8051482fa92e609c617ad1b1 is 50, key is test_row_0/A:col10/1731435239130/Put/seqid=0 2024-11-12T18:13:59,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435299160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435299192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741840_1016 (size=16681) 2024-11-12T18:13:59,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/14775c1f8051482fa92e609c617ad1b1 2024-11-12T18:13:59,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435299201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435299206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435299211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/67e3e14bc3f74b2999c6448e76fc1bc2 is 50, key is test_row_0/B:col10/1731435239130/Put/seqid=0 2024-11-12T18:13:59,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:59,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435299327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741841_1017 (size=12001) 2024-11-12T18:13:59,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/67e3e14bc3f74b2999c6448e76fc1bc2 2024-11-12T18:13:59,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435299334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/751bd946fa2340acaa85859ae495ce1d is 50, key is test_row_0/C:col10/1731435239130/Put/seqid=0 2024-11-12T18:13:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:13:59,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435299376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435299376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435299385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741842_1018 (size=12001) 2024-11-12T18:13:59,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/751bd946fa2340acaa85859ae495ce1d 2024-11-12T18:13:59,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:59,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:59,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/14775c1f8051482fa92e609c617ad1b1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/14775c1f8051482fa92e609c617ad1b1 2024-11-12T18:13:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/14775c1f8051482fa92e609c617ad1b1, entries=250, sequenceid=51, filesize=16.3 K 2024-11-12T18:13:59,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/67e3e14bc3f74b2999c6448e76fc1bc2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67e3e14bc3f74b2999c6448e76fc1bc2 2024-11-12T18:13:59,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67e3e14bc3f74b2999c6448e76fc1bc2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T18:13:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/751bd946fa2340acaa85859ae495ce1d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/751bd946fa2340acaa85859ae495ce1d 2024-11-12T18:13:59,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/751bd946fa2340acaa85859ae495ce1d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T18:13:59,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for 64a54f5030b48a2f7caee1e78dcb2577 in 383ms, sequenceid=51, compaction requested=false 2024-11-12T18:13:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:13:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:59,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:13:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:13:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:13:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:13:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:13:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:13:59,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:13:59,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/8e0b6a49d7fc40fca6f0c79ead0293b1 is 50, key is test_row_0/A:col10/1731435239185/Put/seqid=0 2024-11-12T18:13:59,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:59,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:59,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741843_1019 (size=14341) 2024-11-12T18:13:59,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/8e0b6a49d7fc40fca6f0c79ead0293b1 2024-11-12T18:13:59,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/b0a0282cbe73436d9f8c8de9fe8d44bb is 50, key is test_row_0/B:col10/1731435239185/Put/seqid=0 2024-11-12T18:13:59,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741844_1020 (size=12001) 2024-11-12T18:13:59,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/b0a0282cbe73436d9f8c8de9fe8d44bb 2024-11-12T18:13:59,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435239685, max timestamp=1731435239723Num uniques 859; 2024-11-12T18:13:59,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:13:59,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435239685, max timestamp=1731435239723Num uniques 859; 2024-11-12T18:13:59,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:13:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435239685, max timestamp=1731435239723Num uniques 859; 2024-11-12T18:13:59,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:13:59,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f414852a274241e7854bb74d3d3cc251 is 50, key is test_row_0/C:col10/1731435239185/Put/seqid=0 2024-11-12T18:13:59,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741845_1021 (size=12001) 2024-11-12T18:13:59,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f414852a274241e7854bb74d3d3cc251 2024-11-12T18:13:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435299812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:13:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435299815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:13:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:13:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:13:59,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731435239792, max timestamp=1731435239829Num uniques 1018; 2024-11-12T18:13:59,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731435239792, max timestamp=1731435239829Num uniques 1018; 2024-11-12T18:13:59,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T18:13:59,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T18:13:59,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731435239792, max timestamp=1731435239829Num uniques 1018; 2024-11-12T18:13:59,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/8e0b6a49d7fc40fca6f0c79ead0293b1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/8e0b6a49d7fc40fca6f0c79ead0293b1 2024-11-12T18:13:59,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T18:13:59,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/8e0b6a49d7fc40fca6f0c79ead0293b1, entries=200, sequenceid=61, filesize=14.0 K 2024-11-12T18:13:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/b0a0282cbe73436d9f8c8de9fe8d44bb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b0a0282cbe73436d9f8c8de9fe8d44bb 2024-11-12T18:13:59,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b0a0282cbe73436d9f8c8de9fe8d44bb, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T18:13:59,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f414852a274241e7854bb74d3d3cc251 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f414852a274241e7854bb74d3d3cc251 2024-11-12T18:13:59,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:13:59,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:13:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:13:59,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f414852a274241e7854bb74d3d3cc251, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T18:13:59,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 64a54f5030b48a2f7caee1e78dcb2577 in 374ms, sequenceid=61, compaction requested=true 2024-11-12T18:13:59,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:13:59,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:13:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:13:59,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:13:59,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:13:59,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:13:59,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:13:59,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:13:59,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-12T18:13:59,970 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:13:59,970 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:13:59,975 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:13:59,975 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:13:59,976 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:13:59,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731435239953, max timestamp=1731435239966Num uniques 1077; 2024-11-12T18:13:59,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731435239953, max timestamp=1731435239966Num uniques 1077; 2024-11-12T18:13:59,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T18:13:59,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T18:13:59,977 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:13:59,977 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,977 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:13:59,977 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7d0fee488e794bfb8105cc765c3404a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/14775c1f8051482fa92e609c617ad1b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/8e0b6a49d7fc40fca6f0c79ead0293b1] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=42.0 K 2024-11-12T18:13:59,977 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e4b4de17ffa54271a55849e7f626bb22, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67e3e14bc3f74b2999c6448e76fc1bc2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b0a0282cbe73436d9f8c8de9fe8d44bb] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=35.2 K 2024-11-12T18:13:59,979 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0fee488e794bfb8105cc765c3404a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731435238287 2024-11-12T18:13:59,979 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b4de17ffa54271a55849e7f626bb22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731435238287 2024-11-12T18:13:59,980 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14775c1f8051482fa92e609c617ad1b1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435238740 2024-11-12T18:13:59,980 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e3e14bc3f74b2999c6448e76fc1bc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435238764 2024-11-12T18:13:59,981 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e0b6a49d7fc40fca6f0c79ead0293b1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435239185 2024-11-12T18:13:59,981 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a0282cbe73436d9f8c8de9fe8d44bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435239185 2024-11-12T18:13:59,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731435239953, max timestamp=1731435239966Num uniques 1077; 2024-11-12T18:13:59,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T18:13:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:13:59,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:13:59,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:13:59,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:13:59,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:13:59,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:00,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/35f88f3867364cf681422395a0d200a2 is 50, key is test_row_0/A:col10/1731435239966/Put/seqid=0 2024-11-12T18:14:00,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741846_1022 (size=12001) 2024-11-12T18:14:00,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/35f88f3867364cf681422395a0d200a2 2024-11-12T18:14:00,059 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#10 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:00,060 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/f1115d2d251a41dfa8bb63324d0a2dcd is 50, key is test_row_0/A:col10/1731435239185/Put/seqid=0 2024-11-12T18:14:00,076 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#11 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:00,077 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ca0b10d38f1e40b3a1c21b10e9ebfd21 is 50, key is test_row_0/B:col10/1731435239185/Put/seqid=0 2024-11-12T18:14:00,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741847_1023 (size=12104) 2024-11-12T18:14:00,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ad509a878214477eacaf69a044836c24 is 50, key is test_row_0/B:col10/1731435239966/Put/seqid=0 2024-11-12T18:14:00,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:00,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:00,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435240041, max timestamp=1731435240106Num uniques 859; 2024-11-12T18:14:00,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:00,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435240041, max timestamp=1731435240106Num uniques 859; 2024-11-12T18:14:00,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435240041, max timestamp=1731435240106Num uniques 859; 2024-11-12T18:14:00,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:00,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:00,127 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/f1115d2d251a41dfa8bb63324d0a2dcd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f1115d2d251a41dfa8bb63324d0a2dcd 2024-11-12T18:14:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741848_1024 (size=12104) 2024-11-12T18:14:00,180 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into f1115d2d251a41dfa8bb63324d0a2dcd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:00,180 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:00,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741849_1025 (size=12001) 2024-11-12T18:14:00,183 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435239963; duration=0sec 2024-11-12T18:14:00,184 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:00,184 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:00,184 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:00,192 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:00,192 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:00,193 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,193 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/aa3e1b84d5e649f8b51ec88715692af3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/751bd946fa2340acaa85859ae495ce1d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f414852a274241e7854bb74d3d3cc251] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=35.2 K 2024-11-12T18:14:00,195 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa3e1b84d5e649f8b51ec88715692af3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731435238287 2024-11-12T18:14:00,196 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751bd946fa2340acaa85859ae495ce1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435238764 2024-11-12T18:14:00,197 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f414852a274241e7854bb74d3d3cc251, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435239185 2024-11-12T18:14:00,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731435240118, max timestamp=1731435240231Num uniques 1018; 2024-11-12T18:14:00,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731435240118, max timestamp=1731435240231Num uniques 1018; 2024-11-12T18:14:00,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T18:14:00,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T18:14:00,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731435240118, max timestamp=1731435240231Num uniques 1018; 2024-11-12T18:14:00,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T18:14:00,248 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:00,249 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/3ddb951da62a4db697a271c9945d5380 is 50, key is test_row_0/C:col10/1731435239185/Put/seqid=0 2024-11-12T18:14:00,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741850_1026 (size=12104) 2024-11-12T18:14:00,313 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/3ddb951da62a4db697a271c9945d5380 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/3ddb951da62a4db697a271c9945d5380 2024-11-12T18:14:00,332 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 3ddb951da62a4db697a271c9945d5380(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:00,332 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:00,332 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435239968; duration=0sec 2024-11-12T18:14:00,332 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:00,333 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:00,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435300327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435300331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435300334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731435240317, max timestamp=1731435240337Num uniques 1127; 2024-11-12T18:14:00,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T18:14:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731435240317, max timestamp=1731435240337Num uniques 1127; 2024-11-12T18:14:00,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T18:14:00,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731435240317, max timestamp=1731435240337Num uniques 1127; 2024-11-12T18:14:00,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T18:14:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:14:00,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731435240447, max timestamp=1731435240510Num uniques 1036; 2024-11-12T18:14:00,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-12T18:14:00,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731435240447, max timestamp=1731435240510Num uniques 1036; 2024-11-12T18:14:00,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-12T18:14:00,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731435240447, max timestamp=1731435240510Num uniques 1036; 2024-11-12T18:14:00,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-12T18:14:00,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ad509a878214477eacaf69a044836c24 2024-11-12T18:14:00,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,598 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ca0b10d38f1e40b3a1c21b10e9ebfd21 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ca0b10d38f1e40b3a1c21b10e9ebfd21 2024-11-12T18:14:00,624 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into ca0b10d38f1e40b3a1c21b10e9ebfd21(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:00,624 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:00,624 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435239967; duration=0sec 2024-11-12T18:14:00,624 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:00,624 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:00,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f6b5283c57db4f61891f3df784b5a94f is 50, key is test_row_0/C:col10/1731435239966/Put/seqid=0 2024-11-12T18:14:00,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731435240502, max timestamp=1731435240631Num uniques 1145; 2024-11-12T18:14:00,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-12T18:14:00,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731435240502, max timestamp=1731435240631Num uniques 1145; 2024-11-12T18:14:00,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-12T18:14:00,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731435240502, max timestamp=1731435240631Num uniques 1145; 2024-11-12T18:14:00,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-12T18:14:00,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741851_1027 (size=12001) 2024-11-12T18:14:00,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f6b5283c57db4f61891f3df784b5a94f 2024-11-12T18:14:00,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/35f88f3867364cf681422395a0d200a2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/35f88f3867364cf681422395a0d200a2 2024-11-12T18:14:00,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/35f88f3867364cf681422395a0d200a2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T18:14:00,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435300748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435300753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731435240650, max timestamp=1731435240765Num uniques 1204; 2024-11-12T18:14:00,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-12T18:14:00,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731435240650, max timestamp=1731435240765Num uniques 1204; 2024-11-12T18:14:00,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-12T18:14:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ad509a878214477eacaf69a044836c24 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ad509a878214477eacaf69a044836c24 2024-11-12T18:14:00,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731435240650, max timestamp=1731435240774Num uniques 1204; 2024-11-12T18:14:00,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-12T18:14:00,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ad509a878214477eacaf69a044836c24, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T18:14:00,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f6b5283c57db4f61891f3df784b5a94f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f6b5283c57db4f61891f3df784b5a94f 2024-11-12T18:14:00,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f6b5283c57db4f61891f3df784b5a94f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T18:14:00,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~79.73 KB/81648, currentSize=73.80 KB/75570 for 64a54f5030b48a2f7caee1e78dcb2577 in 877ms, sequenceid=116, compaction requested=false 2024-11-12T18:14:00,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:00,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=80.51 KB heapSize=226.08 KB 2024-11-12T18:14:00,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:00,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:00,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:00,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:00,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7e6379db56bf4d98bfc8c46472a5a884 is 50, key is test_row_0/A:col10/1731435240823/Put/seqid=0 2024-11-12T18:14:00,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:00,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:00,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:00,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741852_1028 (size=19321) 2024-11-12T18:14:00,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7e6379db56bf4d98bfc8c46472a5a884 2024-11-12T18:14:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435300956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435300949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435300965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435300975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435300981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:00,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/aaae18d88c46490e9ebf3edcf0a106ac is 50, key is test_row_0/B:col10/1731435240823/Put/seqid=0 2024-11-12T18:14:01,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:01,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435301081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:01,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435301093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741853_1029 (size=12151) 2024-11-12T18:14:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435301103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/aaae18d88c46490e9ebf3edcf0a106ac 2024-11-12T18:14:01,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435301092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435301113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/c96044136be2407f9116e0b4769419d0 is 50, key is test_row_0/C:col10/1731435240823/Put/seqid=0 2024-11-12T18:14:01,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741854_1030 (size=12151) 2024-11-12T18:14:01,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/c96044136be2407f9116e0b4769419d0 2024-11-12T18:14:01,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:01,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7e6379db56bf4d98bfc8c46472a5a884 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e6379db56bf4d98bfc8c46472a5a884 2024-11-12T18:14:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435301294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e6379db56bf4d98bfc8c46472a5a884, entries=300, sequenceid=234, filesize=18.9 K 2024-11-12T18:14:01,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/aaae18d88c46490e9ebf3edcf0a106ac as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aaae18d88c46490e9ebf3edcf0a106ac 2024-11-12T18:14:01,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435301311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435301320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435301321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aaae18d88c46490e9ebf3edcf0a106ac, entries=150, sequenceid=234, filesize=11.9 K 2024-11-12T18:14:01,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/c96044136be2407f9116e0b4769419d0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c96044136be2407f9116e0b4769419d0 2024-11-12T18:14:01,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435301344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c96044136be2407f9116e0b4769419d0, entries=150, sequenceid=234, filesize=11.9 K 2024-11-12T18:14:01,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~226.03 KB/231456, currentSize=100.63 KB/103050 for 64a54f5030b48a2f7caee1e78dcb2577 in 500ms, sequenceid=234, compaction requested=true 2024-11-12T18:14:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:01,353 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:01,353 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:01,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:01,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:01,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:01,356 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43426 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:01,356 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:01,356 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,356 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f1115d2d251a41dfa8bb63324d0a2dcd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/35f88f3867364cf681422395a0d200a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e6379db56bf4d98bfc8c46472a5a884] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=42.4 K 2024-11-12T18:14:01,357 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:01,357 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:01,357 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,357 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ca0b10d38f1e40b3a1c21b10e9ebfd21, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ad509a878214477eacaf69a044836c24, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aaae18d88c46490e9ebf3edcf0a106ac] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=35.4 K 2024-11-12T18:14:01,358 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1115d2d251a41dfa8bb63324d0a2dcd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435239185 2024-11-12T18:14:01,359 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0b10d38f1e40b3a1c21b10e9ebfd21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435239185 2024-11-12T18:14:01,359 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f88f3867364cf681422395a0d200a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731435239957 2024-11-12T18:14:01,361 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6379db56bf4d98bfc8c46472a5a884, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731435240765 2024-11-12T18:14:01,362 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ad509a878214477eacaf69a044836c24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731435239957 2024-11-12T18:14:01,363 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting aaae18d88c46490e9ebf3edcf0a106ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731435240765 2024-11-12T18:14:01,395 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#18 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:01,397 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/40f4ad51956e4043b082a465219db6ae is 50, key is test_row_0/A:col10/1731435240823/Put/seqid=0 2024-11-12T18:14:01,400 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#19 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:01,402 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e16f78dd1c6d4b7aac00f79987dc48d7 is 50, key is test_row_0/B:col10/1731435240823/Put/seqid=0 2024-11-12T18:14:01,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741855_1031 (size=12357) 2024-11-12T18:14:01,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T18:14:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,422 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:14:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:01,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741856_1032 (size=12357) 2024-11-12T18:14:01,472 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/40f4ad51956e4043b082a465219db6ae as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/40f4ad51956e4043b082a465219db6ae 2024-11-12T18:14:01,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/64c3eb733c4e42c69ff471672c98732f is 50, key is test_row_0/A:col10/1731435240929/Put/seqid=0 2024-11-12T18:14:01,497 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e16f78dd1c6d4b7aac00f79987dc48d7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e16f78dd1c6d4b7aac00f79987dc48d7 2024-11-12T18:14:01,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741857_1033 (size=12151) 2024-11-12T18:14:01,517 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 40f4ad51956e4043b082a465219db6ae(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:01,517 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:01,517 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435241353; duration=0sec 2024-11-12T18:14:01,518 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:01,518 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:01,518 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:01,521 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into e16f78dd1c6d4b7aac00f79987dc48d7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:01,521 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:01,522 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435241353; duration=0sec 2024-11-12T18:14:01,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:01,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:01,529 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:01,529 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:01,529 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:01,529 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/3ddb951da62a4db697a271c9945d5380, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f6b5283c57db4f61891f3df784b5a94f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c96044136be2407f9116e0b4769419d0] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=35.4 K 2024-11-12T18:14:01,531 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ddb951da62a4db697a271c9945d5380, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435239185 2024-11-12T18:14:01,534 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b5283c57db4f61891f3df784b5a94f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731435239957 2024-11-12T18:14:01,536 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c96044136be2407f9116e0b4769419d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731435240765 2024-11-12T18:14:01,586 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:01,593 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d5c307c4763f45339a730b159d6c5d7a is 50, key is test_row_0/C:col10/1731435240823/Put/seqid=0 2024-11-12T18:14:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:01,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741858_1034 (size=12357) 2024-11-12T18:14:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435301677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435301687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435301695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435301698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435301705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435301810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435301825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435301826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435301826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435301826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:01,913 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/64c3eb733c4e42c69ff471672c98732f 2024-11-12T18:14:01,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ea8766f7e47b4f3ab849f92b79489ae4 is 50, key is test_row_0/B:col10/1731435240929/Put/seqid=0 2024-11-12T18:14:01,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741859_1035 (size=12151) 2024-11-12T18:14:01,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ea8766f7e47b4f3ab849f92b79489ae4 2024-11-12T18:14:02,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435302021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/1ee71b841ded4400aab911aa62af5dfd is 50, key is test_row_0/C:col10/1731435240929/Put/seqid=0 2024-11-12T18:14:02,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435302041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435302041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435302042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435302051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741860_1036 (size=12151) 2024-11-12T18:14:02,076 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/1ee71b841ded4400aab911aa62af5dfd 2024-11-12T18:14:02,077 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d5c307c4763f45339a730b159d6c5d7a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d5c307c4763f45339a730b159d6c5d7a 2024-11-12T18:14:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/64c3eb733c4e42c69ff471672c98732f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/64c3eb733c4e42c69ff471672c98732f 2024-11-12T18:14:02,110 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/64c3eb733c4e42c69ff471672c98732f, entries=150, sequenceid=252, filesize=11.9 K 2024-11-12T18:14:02,112 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into d5c307c4763f45339a730b159d6c5d7a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:02,112 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ea8766f7e47b4f3ab849f92b79489ae4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ea8766f7e47b4f3ab849f92b79489ae4 2024-11-12T18:14:02,112 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435241354; duration=0sec 2024-11-12T18:14:02,114 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:02,114 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:02,124 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ea8766f7e47b4f3ab849f92b79489ae4, entries=150, sequenceid=252, filesize=11.9 K 2024-11-12T18:14:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/1ee71b841ded4400aab911aa62af5dfd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1ee71b841ded4400aab911aa62af5dfd 2024-11-12T18:14:02,139 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1ee71b841ded4400aab911aa62af5dfd, entries=150, sequenceid=252, filesize=11.9 K 2024-11-12T18:14:02,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 64a54f5030b48a2f7caee1e78dcb2577 in 720ms, sequenceid=252, compaction requested=false 2024-11-12T18:14:02,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T18:14:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T18:14:02,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T18:14:02,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.9020 sec 2024-11-12T18:14:02,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8800 sec 2024-11-12T18:14:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:02,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:02,349 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:14:02,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/84ad10c6001248078201f7033dc180fd is 50, key is test_row_0/A:col10/1731435241687/Put/seqid=0 2024-11-12T18:14:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T18:14:02,442 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:02,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T18:14:02,447 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:02,450 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:02,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:02,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435302412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435302413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741861_1037 (size=14741) 2024-11-12T18:14:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T18:14:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435302442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435302462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/84ad10c6001248078201f7033dc180fd 2024-11-12T18:14:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435302469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/f2c5487c58e34132bac1dcb970b80b33 is 50, key is test_row_0/B:col10/1731435241687/Put/seqid=0 2024-11-12T18:14:02,553 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T18:14:02,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435302561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741862_1038 (size=12301) 2024-11-12T18:14:02,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/f2c5487c58e34132bac1dcb970b80b33 2024-11-12T18:14:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T18:14:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435302573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435302582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435302582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435302583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/085ae8ffc106468880dd1eda47f50460 is 50, key is test_row_0/C:col10/1731435241687/Put/seqid=0 2024-11-12T18:14:02,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T18:14:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:02,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741863_1039 (size=12301) 2024-11-12T18:14:02,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/085ae8ffc106468880dd1eda47f50460 2024-11-12T18:14:02,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/84ad10c6001248078201f7033dc180fd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/84ad10c6001248078201f7033dc180fd 2024-11-12T18:14:02,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/84ad10c6001248078201f7033dc180fd, entries=200, sequenceid=270, filesize=14.4 K 2024-11-12T18:14:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/f2c5487c58e34132bac1dcb970b80b33 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f2c5487c58e34132bac1dcb970b80b33 2024-11-12T18:14:02,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f2c5487c58e34132bac1dcb970b80b33, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T18:14:02,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/085ae8ffc106468880dd1eda47f50460 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/085ae8ffc106468880dd1eda47f50460 2024-11-12T18:14:02,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/085ae8ffc106468880dd1eda47f50460, entries=150, sequenceid=270, filesize=12.0 K 2024-11-12T18:14:02,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 64a54f5030b48a2f7caee1e78dcb2577 in 398ms, sequenceid=270, compaction requested=true 2024-11-12T18:14:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:02,744 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:02,744 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:02,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:02,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:02,746 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:02,747 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:02,747 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,747 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/40f4ad51956e4043b082a465219db6ae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/64c3eb733c4e42c69ff471672c98732f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/84ad10c6001248078201f7033dc180fd] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=38.3 K 2024-11-12T18:14:02,748 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36809 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:02,748 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:02,748 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,748 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f4ad51956e4043b082a465219db6ae, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731435240765 2024-11-12T18:14:02,748 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e16f78dd1c6d4b7aac00f79987dc48d7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ea8766f7e47b4f3ab849f92b79489ae4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f2c5487c58e34132bac1dcb970b80b33] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=35.9 K 2024-11-12T18:14:02,749 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c3eb733c4e42c69ff471672c98732f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731435240929 2024-11-12T18:14:02,749 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e16f78dd1c6d4b7aac00f79987dc48d7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731435240765 2024-11-12T18:14:02,750 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8766f7e47b4f3ab849f92b79489ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731435240929 2024-11-12T18:14:02,750 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c5487c58e34132bac1dcb970b80b33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731435241687 2024-11-12T18:14:02,753 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ad10c6001248078201f7033dc180fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731435241679 2024-11-12T18:14:02,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T18:14:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,774 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:14:02,776 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T18:14:02,790 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/30650d44956e4f8dbd25d8e1f8463d32 is 50, key is test_row_0/B:col10/1731435241687/Put/seqid=0 2024-11-12T18:14:02,792 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:02,793 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d0fe28631b3547efaa90353ddbb79114 is 50, key is test_row_0/A:col10/1731435241687/Put/seqid=0 2024-11-12T18:14:02,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/fed7ee360dc043179e07e79cf63cf5eb is 50, key is test_row_0/A:col10/1731435242424/Put/seqid=0 2024-11-12T18:14:02,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741864_1040 (size=12609) 2024-11-12T18:14:02,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435302837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435302845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435302841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435302853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T18:14:02,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T18:14:02,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741865_1041 (size=12609) 2024-11-12T18:14:02,877 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T18:14:02,877 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T18:14:02,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T18:14:02,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T18:14:02,879 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T18:14:02,879 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T18:14:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435302862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741866_1042 (size=14741) 2024-11-12T18:14:02,907 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d0fe28631b3547efaa90353ddbb79114 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d0fe28631b3547efaa90353ddbb79114 2024-11-12T18:14:02,925 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into d0fe28631b3547efaa90353ddbb79114(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:02,925 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:02,926 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435242744; duration=0sec 2024-11-12T18:14:02,926 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:02,926 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:02,926 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:02,930 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36809 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:02,930 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:02,930 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:02,930 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d5c307c4763f45339a730b159d6c5d7a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1ee71b841ded4400aab911aa62af5dfd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/085ae8ffc106468880dd1eda47f50460] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=35.9 K 2024-11-12T18:14:02,932 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5c307c4763f45339a730b159d6c5d7a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731435240765 2024-11-12T18:14:02,933 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee71b841ded4400aab911aa62af5dfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731435240929 2024-11-12T18:14:02,935 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085ae8ffc106468880dd1eda47f50460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731435241687 2024-11-12T18:14:02,960 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:02,961 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/29769615ea8f415b8d2743d286926fc8 is 50, key is test_row_0/C:col10/1731435241687/Put/seqid=0 2024-11-12T18:14:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435302971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435302971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435302971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435302980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435302991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741867_1043 (size=12609) 2024-11-12T18:14:03,027 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/29769615ea8f415b8d2743d286926fc8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/29769615ea8f415b8d2743d286926fc8 2024-11-12T18:14:03,046 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 29769615ea8f415b8d2743d286926fc8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:03,046 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:03,046 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435242744; duration=0sec 2024-11-12T18:14:03,047 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:03,047 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T18:14:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435303191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435303192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435303191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435303192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435303211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,271 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/30650d44956e4f8dbd25d8e1f8463d32 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30650d44956e4f8dbd25d8e1f8463d32 2024-11-12T18:14:03,284 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 30650d44956e4f8dbd25d8e1f8463d32(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:03,284 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:03,284 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435242744; duration=0sec 2024-11-12T18:14:03,284 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:03,284 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:03,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/fed7ee360dc043179e07e79cf63cf5eb 2024-11-12T18:14:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/b2e89678ba86470ab11996d1af5f1225 is 50, key is test_row_0/B:col10/1731435242424/Put/seqid=0 2024-11-12T18:14:03,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741868_1044 (size=12301) 2024-11-12T18:14:03,356 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/b2e89678ba86470ab11996d1af5f1225 2024-11-12T18:14:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/cc58549414a041f8b6fe5340d17d78a2 is 50, key is test_row_0/C:col10/1731435242424/Put/seqid=0 2024-11-12T18:14:03,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741869_1045 (size=12301) 2024-11-12T18:14:03,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435303511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435303516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435303517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435303521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435303531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T18:14:03,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/cc58549414a041f8b6fe5340d17d78a2 2024-11-12T18:14:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/fed7ee360dc043179e07e79cf63cf5eb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fed7ee360dc043179e07e79cf63cf5eb 2024-11-12T18:14:03,835 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fed7ee360dc043179e07e79cf63cf5eb, entries=200, sequenceid=289, filesize=14.4 K 2024-11-12T18:14:03,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/b2e89678ba86470ab11996d1af5f1225 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b2e89678ba86470ab11996d1af5f1225 2024-11-12T18:14:03,859 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b2e89678ba86470ab11996d1af5f1225, entries=150, sequenceid=289, filesize=12.0 K 2024-11-12T18:14:03,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/cc58549414a041f8b6fe5340d17d78a2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc58549414a041f8b6fe5340d17d78a2 2024-11-12T18:14:03,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc58549414a041f8b6fe5340d17d78a2, entries=150, sequenceid=289, filesize=12.0 K 2024-11-12T18:14:03,900 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 64a54f5030b48a2f7caee1e78dcb2577 in 1125ms, sequenceid=289, compaction requested=false 2024-11-12T18:14:03,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:03,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:03,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T18:14:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T18:14:03,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T18:14:03,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-11-12T18:14:03,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4650 sec 2024-11-12T18:14:04,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:04,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/272bc79ebcc742d9ae0c1b52e6165bff is 50, key is test_row_0/A:col10/1731435244046/Put/seqid=0 2024-11-12T18:14:04,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741870_1046 (size=12301) 2024-11-12T18:14:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435304099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435304113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/272bc79ebcc742d9ae0c1b52e6165bff 2024-11-12T18:14:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435304130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435304130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435304178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/23c372e33c9640fdbb2ab2232759d28c is 50, key is test_row_0/B:col10/1731435244046/Put/seqid=0 2024-11-12T18:14:04,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741871_1047 (size=12301) 2024-11-12T18:14:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435304282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435304284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435304288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435304291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435304296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435304490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435304501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435304510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435304501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435304510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T18:14:04,630 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:04,633 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T18:14:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T18:14:04,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:04,639 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:04,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:04,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/23c372e33c9640fdbb2ab2232759d28c 2024-11-12T18:14:04,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/4560412e524248c9911b35d037c4ee1e is 50, key is test_row_0/C:col10/1731435244046/Put/seqid=0 2024-11-12T18:14:04,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741872_1048 (size=12301) 2024-11-12T18:14:04,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/4560412e524248c9911b35d037c4ee1e 2024-11-12T18:14:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/272bc79ebcc742d9ae0c1b52e6165bff as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/272bc79ebcc742d9ae0c1b52e6165bff 2024-11-12T18:14:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T18:14:04,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/272bc79ebcc742d9ae0c1b52e6165bff, entries=150, sequenceid=309, filesize=12.0 K 2024-11-12T18:14:04,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/23c372e33c9640fdbb2ab2232759d28c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/23c372e33c9640fdbb2ab2232759d28c 2024-11-12T18:14:04,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/23c372e33c9640fdbb2ab2232759d28c, entries=150, sequenceid=309, filesize=12.0 K 2024-11-12T18:14:04,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/4560412e524248c9911b35d037c4ee1e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/4560412e524248c9911b35d037c4ee1e 2024-11-12T18:14:04,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/4560412e524248c9911b35d037c4ee1e, entries=150, sequenceid=309, filesize=12.0 K 2024-11-12T18:14:04,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 64a54f5030b48a2f7caee1e78dcb2577 in 719ms, sequenceid=309, compaction requested=true 2024-11-12T18:14:04,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:04,766 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:04,766 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:04,768 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:04,768 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:04,768 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:04,768 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:04,768 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:04,768 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:04,769 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30650d44956e4f8dbd25d8e1f8463d32, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b2e89678ba86470ab11996d1af5f1225, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/23c372e33c9640fdbb2ab2232759d28c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.3 K 2024-11-12T18:14:04,769 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d0fe28631b3547efaa90353ddbb79114, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fed7ee360dc043179e07e79cf63cf5eb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/272bc79ebcc742d9ae0c1b52e6165bff] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=38.7 K 2024-11-12T18:14:04,769 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 30650d44956e4f8dbd25d8e1f8463d32, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731435241687 2024-11-12T18:14:04,770 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e89678ba86470ab11996d1af5f1225, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731435242424 2024-11-12T18:14:04,770 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0fe28631b3547efaa90353ddbb79114, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731435241687 2024-11-12T18:14:04,771 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c372e33c9640fdbb2ab2232759d28c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731435244037 2024-11-12T18:14:04,771 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed7ee360dc043179e07e79cf63cf5eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731435242395 2024-11-12T18:14:04,777 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272bc79ebcc742d9ae0c1b52e6165bff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731435244037 2024-11-12T18:14:04,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T18:14:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:04,797 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:04,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:04,808 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:04,809 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/594ae7c967eb48cdad829a8f9d10f018 is 50, key is test_row_0/A:col10/1731435244046/Put/seqid=0 2024-11-12T18:14:04,819 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:04,820 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/6ab75664b82c4c3e94c26acacaae184a is 50, key is test_row_0/B:col10/1731435244046/Put/seqid=0 2024-11-12T18:14:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/9a676def3c1f4146981c3bf4b22904b8 is 50, key is test_row_0/A:col10/1731435244120/Put/seqid=0 2024-11-12T18:14:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741874_1050 (size=12711) 2024-11-12T18:14:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741873_1049 (size=12711) 2024-11-12T18:14:04,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741875_1051 (size=12301) 2024-11-12T18:14:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435304863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435304863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435304877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435304885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435304889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T18:14:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435304991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435304991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435305000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435305011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435305011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435305210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435305210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435305221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435305221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435305221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T18:14:05,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/9a676def3c1f4146981c3bf4b22904b8 2024-11-12T18:14:05,295 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/594ae7c967eb48cdad829a8f9d10f018 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/594ae7c967eb48cdad829a8f9d10f018 2024-11-12T18:14:05,296 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/6ab75664b82c4c3e94c26acacaae184a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/6ab75664b82c4c3e94c26acacaae184a 2024-11-12T18:14:05,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/813a090b49c64fa2adbb119a1303970d is 50, key is test_row_0/B:col10/1731435244120/Put/seqid=0 2024-11-12T18:14:05,324 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 6ab75664b82c4c3e94c26acacaae184a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:05,324 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:05,325 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435244766; duration=0sec 2024-11-12T18:14:05,325 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:05,325 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:05,325 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:05,326 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 594ae7c967eb48cdad829a8f9d10f018(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:05,326 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:05,326 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435244766; duration=0sec 2024-11-12T18:14:05,326 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:05,326 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:05,327 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:05,327 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:05,328 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:05,328 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/29769615ea8f415b8d2743d286926fc8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc58549414a041f8b6fe5340d17d78a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/4560412e524248c9911b35d037c4ee1e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.3 K 2024-11-12T18:14:05,328 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 29769615ea8f415b8d2743d286926fc8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731435241687 2024-11-12T18:14:05,329 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting cc58549414a041f8b6fe5340d17d78a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731435242424 2024-11-12T18:14:05,332 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4560412e524248c9911b35d037c4ee1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731435244037 2024-11-12T18:14:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741876_1052 (size=12301) 2024-11-12T18:14:05,349 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/813a090b49c64fa2adbb119a1303970d 2024-11-12T18:14:05,366 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:05,367 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/251a968a6e864a6a93eba2ad26f7442e is 50, key is test_row_0/C:col10/1731435244046/Put/seqid=0 2024-11-12T18:14:05,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/c2b95dab00b04547ba6b4e613ef5a12c is 50, key is test_row_0/C:col10/1731435244120/Put/seqid=0 2024-11-12T18:14:05,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741878_1054 (size=12301) 2024-11-12T18:14:05,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741877_1053 (size=12711) 2024-11-12T18:14:05,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435305520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435305521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435305530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435305530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435305530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T18:14:05,806 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/c2b95dab00b04547ba6b4e613ef5a12c 2024-11-12T18:14:05,816 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/251a968a6e864a6a93eba2ad26f7442e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/251a968a6e864a6a93eba2ad26f7442e 2024-11-12T18:14:05,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/9a676def3c1f4146981c3bf4b22904b8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/9a676def3c1f4146981c3bf4b22904b8 2024-11-12T18:14:05,826 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/9a676def3c1f4146981c3bf4b22904b8, entries=150, sequenceid=324, filesize=12.0 K 2024-11-12T18:14:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/813a090b49c64fa2adbb119a1303970d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/813a090b49c64fa2adbb119a1303970d 2024-11-12T18:14:05,829 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 251a968a6e864a6a93eba2ad26f7442e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:05,829 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:05,829 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435244766; duration=0sec 2024-11-12T18:14:05,829 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:05,829 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:05,839 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/813a090b49c64fa2adbb119a1303970d, entries=150, sequenceid=324, filesize=12.0 K 2024-11-12T18:14:05,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/c2b95dab00b04547ba6b4e613ef5a12c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c2b95dab00b04547ba6b4e613ef5a12c 2024-11-12T18:14:05,848 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c2b95dab00b04547ba6b4e613ef5a12c, entries=150, sequenceid=324, filesize=12.0 K 2024-11-12T18:14:05,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 64a54f5030b48a2f7caee1e78dcb2577 in 1053ms, sequenceid=324, compaction requested=false 2024-11-12T18:14:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T18:14:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T18:14:05,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T18:14:05,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2280 sec 2024-11-12T18:14:05,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2170 sec 2024-11-12T18:14:06,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:14:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:06,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:06,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:06,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/e09c8943c5d44acc8138e3dc972d480b is 50, key is test_row_0/A:col10/1731435246032/Put/seqid=0 2024-11-12T18:14:06,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435306053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435306054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435306057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435306058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435306062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741879_1055 (size=12301) 2024-11-12T18:14:06,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/e09c8943c5d44acc8138e3dc972d480b 2024-11-12T18:14:06,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/f49ee59899e14cc98c961831b7a6f747 is 50, key is test_row_0/B:col10/1731435246032/Put/seqid=0 2024-11-12T18:14:06,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741880_1056 (size=12301) 2024-11-12T18:14:06,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/f49ee59899e14cc98c961831b7a6f747 2024-11-12T18:14:06,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435306170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435306170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435306181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435306181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435306181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/b1848a3627934656a0385602de0df341 is 50, key is test_row_0/C:col10/1731435246032/Put/seqid=0 2024-11-12T18:14:06,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741881_1057 (size=12301) 2024-11-12T18:14:06,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435306382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435306382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435306390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435306391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435306391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435306691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435306691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435306710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435306710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435306711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T18:14:06,781 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:06,781 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T18:14:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T18:14:06,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/b1848a3627934656a0385602de0df341 2024-11-12T18:14:06,785 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:06,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:06,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/e09c8943c5d44acc8138e3dc972d480b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e09c8943c5d44acc8138e3dc972d480b 2024-11-12T18:14:06,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e09c8943c5d44acc8138e3dc972d480b, entries=150, sequenceid=347, filesize=12.0 K 2024-11-12T18:14:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/f49ee59899e14cc98c961831b7a6f747 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f49ee59899e14cc98c961831b7a6f747 2024-11-12T18:14:06,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f49ee59899e14cc98c961831b7a6f747, entries=150, sequenceid=347, filesize=12.0 K 2024-11-12T18:14:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/b1848a3627934656a0385602de0df341 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/b1848a3627934656a0385602de0df341 2024-11-12T18:14:06,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/b1848a3627934656a0385602de0df341, entries=150, sequenceid=347, filesize=12.0 K 2024-11-12T18:14:06,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 64a54f5030b48a2f7caee1e78dcb2577 in 799ms, sequenceid=347, compaction requested=true 2024-11-12T18:14:06,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:06,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:06,834 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:06,836 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:06,836 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:06,836 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:06,836 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:06,836 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:06,836 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:06,836 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/594ae7c967eb48cdad829a8f9d10f018, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/9a676def3c1f4146981c3bf4b22904b8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e09c8943c5d44acc8138e3dc972d480b] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.4 K 2024-11-12T18:14:06,836 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/6ab75664b82c4c3e94c26acacaae184a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/813a090b49c64fa2adbb119a1303970d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f49ee59899e14cc98c961831b7a6f747] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.4 K 2024-11-12T18:14:06,837 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 594ae7c967eb48cdad829a8f9d10f018, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731435244037 2024-11-12T18:14:06,837 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab75664b82c4c3e94c26acacaae184a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731435244037 2024-11-12T18:14:06,837 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 813a090b49c64fa2adbb119a1303970d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731435244088 2024-11-12T18:14:06,838 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a676def3c1f4146981c3bf4b22904b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731435244088 2024-11-12T18:14:06,838 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f49ee59899e14cc98c961831b7a6f747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731435244881 2024-11-12T18:14:06,841 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09c8943c5d44acc8138e3dc972d480b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731435244881 2024-11-12T18:14:06,868 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:06,869 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/7ddfa30ef3cb4c439ce7e71770304b58 is 50, key is test_row_0/B:col10/1731435246032/Put/seqid=0 2024-11-12T18:14:06,871 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:06,872 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/925fee6d86e6481286ab654a0dc69e34 is 50, key is test_row_0/A:col10/1731435246032/Put/seqid=0 2024-11-12T18:14:06,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741882_1058 (size=12813) 2024-11-12T18:14:06,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741883_1059 (size=12813) 2024-11-12T18:14:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T18:14:06,893 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/7ddfa30ef3cb4c439ce7e71770304b58 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7ddfa30ef3cb4c439ce7e71770304b58 2024-11-12T18:14:06,897 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/925fee6d86e6481286ab654a0dc69e34 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/925fee6d86e6481286ab654a0dc69e34 2024-11-12T18:14:06,910 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 925fee6d86e6481286ab654a0dc69e34(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:06,910 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:06,910 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435246834; duration=0sec 2024-11-12T18:14:06,910 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:06,910 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:06,910 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:06,913 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:06,913 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:06,914 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:06,914 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/251a968a6e864a6a93eba2ad26f7442e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c2b95dab00b04547ba6b4e613ef5a12c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/b1848a3627934656a0385602de0df341] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.4 K 2024-11-12T18:14:06,914 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 251a968a6e864a6a93eba2ad26f7442e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731435244037 2024-11-12T18:14:06,915 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b95dab00b04547ba6b4e613ef5a12c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731435244088 2024-11-12T18:14:06,916 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1848a3627934656a0385602de0df341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731435244881 2024-11-12T18:14:06,917 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 7ddfa30ef3cb4c439ce7e71770304b58(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:06,917 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:06,917 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435246834; duration=0sec 2024-11-12T18:14:06,917 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:06,917 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:06,929 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:06,930 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/8bc1112dbc9e44cc9a44b9592a91a83c is 50, key is test_row_0/C:col10/1731435246032/Put/seqid=0 2024-11-12T18:14:06,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T18:14:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:06,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741884_1060 (size=12813) 2024-11-12T18:14:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/623af9c99ed740d78fb493bc09de0272 is 50, key is test_row_0/A:col10/1731435246057/Put/seqid=0 2024-11-12T18:14:06,979 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/8bc1112dbc9e44cc9a44b9592a91a83c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/8bc1112dbc9e44cc9a44b9592a91a83c 2024-11-12T18:14:07,060 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 8bc1112dbc9e44cc9a44b9592a91a83c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:07,060 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:07,060 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435246834; duration=0sec 2024-11-12T18:14:07,061 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:07,061 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:07,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741885_1061 (size=12301) 2024-11-12T18:14:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T18:14:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:07,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:07,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435307245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435307249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435307249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435307255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435307255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435307360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435307360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435307360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435307371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435307371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T18:14:07,467 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/623af9c99ed740d78fb493bc09de0272 2024-11-12T18:14:07,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/da307cef80ef43cd853e495d0eee77b1 is 50, key is test_row_0/B:col10/1731435246057/Put/seqid=0 2024-11-12T18:14:07,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435307571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435307571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435307573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741886_1062 (size=12301) 2024-11-12T18:14:07,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435307580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435307580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435307885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435307885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435307885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435307891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435307891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T18:14:07,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/da307cef80ef43cd853e495d0eee77b1 2024-11-12T18:14:08,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/30d5cd9e138b4d659ac9c84660ef7ca2 is 50, key is test_row_0/C:col10/1731435246057/Put/seqid=0 2024-11-12T18:14:08,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741887_1063 (size=12301) 2024-11-12T18:14:08,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435308400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:08,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435308402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:08,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435308402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435308402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435308402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:08,422 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/30d5cd9e138b4d659ac9c84660ef7ca2 2024-11-12T18:14:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/623af9c99ed740d78fb493bc09de0272 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/623af9c99ed740d78fb493bc09de0272 2024-11-12T18:14:08,445 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/623af9c99ed740d78fb493bc09de0272, entries=150, sequenceid=362, filesize=12.0 K 2024-11-12T18:14:08,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/da307cef80ef43cd853e495d0eee77b1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/da307cef80ef43cd853e495d0eee77b1 2024-11-12T18:14:08,455 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/da307cef80ef43cd853e495d0eee77b1, entries=150, sequenceid=362, filesize=12.0 K 2024-11-12T18:14:08,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/30d5cd9e138b4d659ac9c84660ef7ca2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/30d5cd9e138b4d659ac9c84660ef7ca2 2024-11-12T18:14:08,504 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/30d5cd9e138b4d659ac9c84660ef7ca2, entries=150, sequenceid=362, filesize=12.0 K 2024-11-12T18:14:08,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 64a54f5030b48a2f7caee1e78dcb2577 in 1565ms, sequenceid=362, compaction requested=false 2024-11-12T18:14:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T18:14:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T18:14:08,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T18:14:08,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-11-12T18:14:08,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7350 sec 2024-11-12T18:14:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T18:14:08,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:08,932 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T18:14:08,935 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T18:14:08,936 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:08,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T18:14:09,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T18:14:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:09,091 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:14:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/fc4ddb818baa47f0b5a4d43a33eafda4 is 50, key is test_row_0/A:col10/1731435247248/Put/seqid=0 2024-11-12T18:14:09,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741888_1064 (size=12301) 2024-11-12T18:14:09,135 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/fc4ddb818baa47f0b5a4d43a33eafda4 2024-11-12T18:14:09,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ff563f62b26a45bdb1c69069047e31b0 is 50, key is test_row_0/B:col10/1731435247248/Put/seqid=0 2024-11-12T18:14:09,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741889_1065 (size=12301) 2024-11-12T18:14:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T18:14:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:09,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435309461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435309462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435309463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435309463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435309464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T18:14:09,567 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ff563f62b26a45bdb1c69069047e31b0 2024-11-12T18:14:09,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435309580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/97a4553c6f214b1d92ee6092112bd04e is 50, key is test_row_0/C:col10/1731435247248/Put/seqid=0 2024-11-12T18:14:09,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435309581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435309581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435309581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435309581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741890_1066 (size=12301) 2024-11-12T18:14:09,651 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/97a4553c6f214b1d92ee6092112bd04e 2024-11-12T18:14:09,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/fc4ddb818baa47f0b5a4d43a33eafda4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fc4ddb818baa47f0b5a4d43a33eafda4 2024-11-12T18:14:09,683 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fc4ddb818baa47f0b5a4d43a33eafda4, entries=150, sequenceid=382, filesize=12.0 K 2024-11-12T18:14:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/ff563f62b26a45bdb1c69069047e31b0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ff563f62b26a45bdb1c69069047e31b0 2024-11-12T18:14:09,712 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ff563f62b26a45bdb1c69069047e31b0, entries=150, sequenceid=382, filesize=12.0 K 2024-11-12T18:14:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/97a4553c6f214b1d92ee6092112bd04e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/97a4553c6f214b1d92ee6092112bd04e 2024-11-12T18:14:09,743 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/97a4553c6f214b1d92ee6092112bd04e, entries=150, sequenceid=382, filesize=12.0 K 2024-11-12T18:14:09,749 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 64a54f5030b48a2f7caee1e78dcb2577 in 657ms, sequenceid=382, compaction requested=true 2024-11-12T18:14:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T18:14:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T18:14:09,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T18:14:09,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 824 msec 2024-11-12T18:14:09,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 832 msec 2024-11-12T18:14:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:09,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:09,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:09,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:09,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:09,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:09,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/44f0bd788bd34f9ab239b72b7e646d74 is 50, key is test_row_0/A:col10/1731435249461/Put/seqid=0 2024-11-12T18:14:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435309836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435309837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741891_1067 (size=12301) 2024-11-12T18:14:09,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/44f0bd788bd34f9ab239b72b7e646d74 2024-11-12T18:14:09,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435309841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435309842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435309843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/48499a4021364ba196f7b8cad51c797f is 50, key is test_row_0/B:col10/1731435249461/Put/seqid=0 2024-11-12T18:14:09,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741892_1068 (size=12301) 2024-11-12T18:14:09,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/48499a4021364ba196f7b8cad51c797f 2024-11-12T18:14:09,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/7644c3ffcd4a452b9938b42e40315f85 is 50, key is test_row_0/C:col10/1731435249461/Put/seqid=0 2024-11-12T18:14:09,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435309950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435309961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435309962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435309962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435309963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:09,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:09,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741893_1069 (size=12301) 2024-11-12T18:14:09,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/7644c3ffcd4a452b9938b42e40315f85 2024-11-12T18:14:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/44f0bd788bd34f9ab239b72b7e646d74 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/44f0bd788bd34f9ab239b72b7e646d74 2024-11-12T18:14:10,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/44f0bd788bd34f9ab239b72b7e646d74, entries=150, sequenceid=398, filesize=12.0 K 2024-11-12T18:14:10,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/48499a4021364ba196f7b8cad51c797f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/48499a4021364ba196f7b8cad51c797f 2024-11-12T18:14:10,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/48499a4021364ba196f7b8cad51c797f, entries=150, sequenceid=398, filesize=12.0 K 2024-11-12T18:14:10,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/7644c3ffcd4a452b9938b42e40315f85 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7644c3ffcd4a452b9938b42e40315f85 2024-11-12T18:14:10,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7644c3ffcd4a452b9938b42e40315f85, entries=150, sequenceid=398, filesize=12.0 K 2024-11-12T18:14:10,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 64a54f5030b48a2f7caee1e78dcb2577 in 226ms, sequenceid=398, compaction requested=true 2024-11-12T18:14:10,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:10,032 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:10,032 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:10,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:10,035 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:10,035 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:10,035 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:10,035 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/925fee6d86e6481286ab654a0dc69e34, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/623af9c99ed740d78fb493bc09de0272, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fc4ddb818baa47f0b5a4d43a33eafda4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/44f0bd788bd34f9ab239b72b7e646d74] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=48.6 K 2024-11-12T18:14:10,036 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:10,036 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:10,036 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:10,036 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7ddfa30ef3cb4c439ce7e71770304b58, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/da307cef80ef43cd853e495d0eee77b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ff563f62b26a45bdb1c69069047e31b0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/48499a4021364ba196f7b8cad51c797f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=48.6 K 2024-11-12T18:14:10,037 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ddfa30ef3cb4c439ce7e71770304b58, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731435244881 2024-11-12T18:14:10,037 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 925fee6d86e6481286ab654a0dc69e34, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731435244881 2024-11-12T18:14:10,041 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting da307cef80ef43cd853e495d0eee77b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731435246052 2024-11-12T18:14:10,041 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 623af9c99ed740d78fb493bc09de0272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731435246052 2024-11-12T18:14:10,041 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ff563f62b26a45bdb1c69069047e31b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731435247244 2024-11-12T18:14:10,042 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc4ddb818baa47f0b5a4d43a33eafda4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731435247244 2024-11-12T18:14:10,042 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 48499a4021364ba196f7b8cad51c797f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731435249461 2024-11-12T18:14:10,043 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f0bd788bd34f9ab239b72b7e646d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731435249461 2024-11-12T18:14:10,062 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:10,063 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e0e2fe7c3d794b1580d8c7681bf69997 is 50, key is test_row_0/B:col10/1731435249461/Put/seqid=0 2024-11-12T18:14:10,072 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T18:14:10,073 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:10,074 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:10,074 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/356483b6b7e647c6a7429869c01e8555 is 50, key is test_row_0/A:col10/1731435249461/Put/seqid=0 2024-11-12T18:14:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T18:14:10,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T18:14:10,078 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:10,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:10,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741894_1070 (size=12949) 2024-11-12T18:14:10,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741895_1071 (size=12949) 2024-11-12T18:14:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:10,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T18:14:10,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:10,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:10,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:10,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:10,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:10,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:10,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7e61dbde5a1a44c4887b29b11d1b52b7 is 50, key is test_row_0/A:col10/1731435250160/Put/seqid=0 2024-11-12T18:14:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T18:14:10,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741896_1072 (size=14741) 2024-11-12T18:14:10,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7e61dbde5a1a44c4887b29b11d1b52b7 2024-11-12T18:14:10,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435310201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435310204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435310198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435310207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435310209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/7077663fe45843cca5806d165f1c90b8 is 50, key is test_row_0/B:col10/1731435250160/Put/seqid=0 2024-11-12T18:14:10,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T18:14:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:10,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:10,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741897_1073 (size=12301) 2024-11-12T18:14:10,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/7077663fe45843cca5806d165f1c90b8 2024-11-12T18:14:10,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/7319ebbdeeb44a789f3f5cc9c10764c1 is 50, key is test_row_0/C:col10/1731435250160/Put/seqid=0 2024-11-12T18:14:10,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741898_1074 (size=12301) 2024-11-12T18:14:10,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/7319ebbdeeb44a789f3f5cc9c10764c1 2024-11-12T18:14:10,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7e61dbde5a1a44c4887b29b11d1b52b7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e61dbde5a1a44c4887b29b11d1b52b7 2024-11-12T18:14:10,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e61dbde5a1a44c4887b29b11d1b52b7, entries=200, sequenceid=415, filesize=14.4 K 2024-11-12T18:14:10,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/7077663fe45843cca5806d165f1c90b8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7077663fe45843cca5806d165f1c90b8 2024-11-12T18:14:10,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7077663fe45843cca5806d165f1c90b8, entries=150, sequenceid=415, filesize=12.0 K 2024-11-12T18:14:10,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435310320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/7319ebbdeeb44a789f3f5cc9c10764c1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7319ebbdeeb44a789f3f5cc9c10764c1 2024-11-12T18:14:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435310320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435310321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435310321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435310321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7319ebbdeeb44a789f3f5cc9c10764c1, entries=150, sequenceid=415, filesize=12.0 K 2024-11-12T18:14:10,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 64a54f5030b48a2f7caee1e78dcb2577 in 170ms, sequenceid=415, compaction requested=true 2024-11-12T18:14:10,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:10,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:10,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:14:10,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:10,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T18:14:10,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:10,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-12T18:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T18:14:10,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T18:14:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:10,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:10,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5487c8b036f946c499884c26f95125e7 is 50, key is test_row_0/A:col10/1731435250200/Put/seqid=0 2024-11-12T18:14:10,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741899_1075 (size=12301) 2024-11-12T18:14:10,508 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e0e2fe7c3d794b1580d8c7681bf69997 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e2fe7c3d794b1580d8c7681bf69997 2024-11-12T18:14:10,520 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into e0e2fe7c3d794b1580d8c7681bf69997(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:10,520 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:10,520 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=12, startTime=1731435250032; duration=0sec 2024-11-12T18:14:10,520 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-12T18:14:10,521 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:10,521 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-12T18:14:10,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:10,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:10,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. because compaction request was cancelled 2024-11-12T18:14:10,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:10,522 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T18:14:10,528 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62017 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T18:14:10,528 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:10,528 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:10,529 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/8bc1112dbc9e44cc9a44b9592a91a83c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/30d5cd9e138b4d659ac9c84660ef7ca2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/97a4553c6f214b1d92ee6092112bd04e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7644c3ffcd4a452b9938b42e40315f85, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7319ebbdeeb44a789f3f5cc9c10764c1] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=60.6 K 2024-11-12T18:14:10,529 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc1112dbc9e44cc9a44b9592a91a83c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731435244881 2024-11-12T18:14:10,530 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d5cd9e138b4d659ac9c84660ef7ca2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731435246052 2024-11-12T18:14:10,534 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a4553c6f214b1d92ee6092112bd04e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731435247244 2024-11-12T18:14:10,535 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/356483b6b7e647c6a7429869c01e8555 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/356483b6b7e647c6a7429869c01e8555 2024-11-12T18:14:10,536 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7644c3ffcd4a452b9938b42e40315f85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731435249461 2024-11-12T18:14:10,537 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7319ebbdeeb44a789f3f5cc9c10764c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731435249837 2024-11-12T18:14:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:10,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:10,554 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 356483b6b7e647c6a7429869c01e8555(size=12.6 K), total size for store is 27.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:10,554 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:10,554 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=12, startTime=1731435250031; duration=0sec 2024-11-12T18:14:10,554 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:14:10,554 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:10,554 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:10,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-12T18:14:10,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:10,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:10,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. because compaction request was cancelled 2024-11-12T18:14:10,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:10,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:14:10,557 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:10,557 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:10,557 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. because compaction request was cancelled 2024-11-12T18:14:10,557 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:10,575 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:10,577 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/e566589145d54246b1f23dcd5ab1b0ec is 50, key is test_row_0/C:col10/1731435250160/Put/seqid=0 2024-11-12T18:14:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435310585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435310582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741900_1076 (size=12983) 2024-11-12T18:14:10,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435310593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435310594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435310594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T18:14:10,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435310701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435310701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435310710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435310711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435310711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,825 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5487c8b036f946c499884c26f95125e7 2024-11-12T18:14:10,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/5b729a75cac64eedad70fa9134efeed8 is 50, key is test_row_0/B:col10/1731435250200/Put/seqid=0 2024-11-12T18:14:10,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741901_1077 (size=12301) 2024-11-12T18:14:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435310911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435310911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435310920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435310920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435310920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,001 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/e566589145d54246b1f23dcd5ab1b0ec as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e566589145d54246b1f23dcd5ab1b0ec 2024-11-12T18:14:11,011 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into e566589145d54246b1f23dcd5ab1b0ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:11,011 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:11,011 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=11, startTime=1731435250333; duration=0sec 2024-11-12T18:14:11,012 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:11,012 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T18:14:11,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435311220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435311220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435311231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435311231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435311232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/5b729a75cac64eedad70fa9134efeed8 2024-11-12T18:14:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/198669fc498540c9ad420c4fdeeade92 is 50, key is test_row_0/C:col10/1731435250200/Put/seqid=0 2024-11-12T18:14:11,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741902_1078 (size=12301) 2024-11-12T18:14:11,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/198669fc498540c9ad420c4fdeeade92 2024-11-12T18:14:11,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5487c8b036f946c499884c26f95125e7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5487c8b036f946c499884c26f95125e7 2024-11-12T18:14:11,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5487c8b036f946c499884c26f95125e7, entries=150, sequenceid=430, filesize=12.0 K 2024-11-12T18:14:11,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/5b729a75cac64eedad70fa9134efeed8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/5b729a75cac64eedad70fa9134efeed8 2024-11-12T18:14:11,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/5b729a75cac64eedad70fa9134efeed8, entries=150, sequenceid=430, filesize=12.0 K 2024-11-12T18:14:11,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/198669fc498540c9ad420c4fdeeade92 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/198669fc498540c9ad420c4fdeeade92 2024-11-12T18:14:11,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/198669fc498540c9ad420c4fdeeade92, entries=150, sequenceid=430, filesize=12.0 K 2024-11-12T18:14:11,346 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 64a54f5030b48a2f7caee1e78dcb2577 in 955ms, sequenceid=430, compaction requested=true 2024-11-12T18:14:11,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:11,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:11,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T18:14:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T18:14:11,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T18:14:11,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2720 sec 2024-11-12T18:14:11,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2820 sec 2024-11-12T18:14:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:11,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:14:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:11,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:11,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d22b528f440746c59e3439d440c8ee38 is 50, key is test_row_0/A:col10/1731435250579/Put/seqid=0 2024-11-12T18:14:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741903_1079 (size=14741) 2024-11-12T18:14:11,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435311755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435311757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435311758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435311758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435311760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435311870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435311870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435311870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435311870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435311870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:11,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435312080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435312081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435312081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435312081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435312082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d22b528f440746c59e3439d440c8ee38 2024-11-12T18:14:12,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/aee3e026f8bc4d47abf5f55875ad523b is 50, key is test_row_0/B:col10/1731435250579/Put/seqid=0 2024-11-12T18:14:12,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741904_1080 (size=12301) 2024-11-12T18:14:12,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/aee3e026f8bc4d47abf5f55875ad523b 2024-11-12T18:14:12,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/cc59dd4a50f14186a96ab580d656d0ba is 50, key is test_row_0/C:col10/1731435250579/Put/seqid=0 2024-11-12T18:14:12,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741905_1081 (size=12301) 2024-11-12T18:14:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T18:14:12,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:12,222 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-12T18:14:12,224 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:12,226 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:12,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T18:14:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T18:14:12,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T18:14:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435312391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435312391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435312391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435312391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435312391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T18:14:12,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T18:14:12,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:12,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/cc59dd4a50f14186a96ab580d656d0ba 2024-11-12T18:14:12,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d22b528f440746c59e3439d440c8ee38 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d22b528f440746c59e3439d440c8ee38 2024-11-12T18:14:12,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d22b528f440746c59e3439d440c8ee38, entries=200, sequenceid=453, filesize=14.4 K 2024-11-12T18:14:12,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/aee3e026f8bc4d47abf5f55875ad523b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aee3e026f8bc4d47abf5f55875ad523b 2024-11-12T18:14:12,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T18:14:12,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:12,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:12,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aee3e026f8bc4d47abf5f55875ad523b, entries=150, sequenceid=453, filesize=12.0 K 2024-11-12T18:14:12,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/cc59dd4a50f14186a96ab580d656d0ba as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc59dd4a50f14186a96ab580d656d0ba 2024-11-12T18:14:12,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc59dd4a50f14186a96ab580d656d0ba, entries=150, sequenceid=453, filesize=12.0 K 2024-11-12T18:14:12,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 64a54f5030b48a2f7caee1e78dcb2577 in 1000ms, sequenceid=453, compaction requested=true 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:12,735 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:12,735 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:12,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:12,737 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:12,737 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:12,738 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,738 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e2fe7c3d794b1580d8c7681bf69997, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7077663fe45843cca5806d165f1c90b8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/5b729a75cac64eedad70fa9134efeed8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aee3e026f8bc4d47abf5f55875ad523b] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=48.7 K 2024-11-12T18:14:12,738 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:12,738 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:12,738 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,738 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e2fe7c3d794b1580d8c7681bf69997, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731435249461 2024-11-12T18:14:12,738 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/356483b6b7e647c6a7429869c01e8555, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e61dbde5a1a44c4887b29b11d1b52b7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5487c8b036f946c499884c26f95125e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d22b528f440746c59e3439d440c8ee38] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=53.4 K 2024-11-12T18:14:12,739 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7077663fe45843cca5806d165f1c90b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731435249837 2024-11-12T18:14:12,739 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356483b6b7e647c6a7429869c01e8555, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731435249461 2024-11-12T18:14:12,741 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b729a75cac64eedad70fa9134efeed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731435250200 2024-11-12T18:14:12,741 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e61dbde5a1a44c4887b29b11d1b52b7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731435249836 2024-11-12T18:14:12,741 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting aee3e026f8bc4d47abf5f55875ad523b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731435250579 2024-11-12T18:14:12,741 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5487c8b036f946c499884c26f95125e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731435250200 2024-11-12T18:14:12,743 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d22b528f440746c59e3439d440c8ee38, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731435250579 2024-11-12T18:14:12,757 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:12,758 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/0c05fbfacb3148d498e9bafce3fbdb54 is 50, key is test_row_0/A:col10/1731435250579/Put/seqid=0 2024-11-12T18:14:12,759 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:12,760 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/364bf0ce7db94aa693dba7e3ff6e1159 is 50, key is test_row_0/B:col10/1731435250579/Put/seqid=0 2024-11-12T18:14:12,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741906_1082 (size=13085) 2024-11-12T18:14:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741907_1083 (size=13085) 2024-11-12T18:14:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T18:14:12,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T18:14:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:12,864 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:12,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d71935f9517f4cfaab14c65be4540401 is 50, key is test_row_0/A:col10/1731435251755/Put/seqid=0 2024-11-12T18:14:12,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741908_1084 (size=12301) 2024-11-12T18:14:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:12,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435312942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435252938, max timestamp=1731435252948Num uniques 859; 2024-11-12T18:14:12,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435252938, max timestamp=1731435252948Num uniques 859; 2024-11-12T18:14:12,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435252938, max timestamp=1731435252948Num uniques 859; 2024-11-12T18:14:12,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:12,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:12,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435312982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435312982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435312984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435312988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435313061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435313101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435313101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435313110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435313110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,207 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/0c05fbfacb3148d498e9bafce3fbdb54 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/0c05fbfacb3148d498e9bafce3fbdb54 2024-11-12T18:14:13,222 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/364bf0ce7db94aa693dba7e3ff6e1159 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/364bf0ce7db94aa693dba7e3ff6e1159 2024-11-12T18:14:13,230 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 0c05fbfacb3148d498e9bafce3fbdb54(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:13,230 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:13,230 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=12, startTime=1731435252735; duration=0sec 2024-11-12T18:14:13,236 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:13,236 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:13,236 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:13,260 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 364bf0ce7db94aa693dba7e3ff6e1159(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:13,260 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:13,261 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=12, startTime=1731435252735; duration=0sec 2024-11-12T18:14:13,261 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:13,261 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:13,264 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:13,265 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:13,265 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:13,265 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e566589145d54246b1f23dcd5ab1b0ec, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/198669fc498540c9ad420c4fdeeade92, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc59dd4a50f14186a96ab580d656d0ba] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.7 K 2024-11-12T18:14:13,269 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e566589145d54246b1f23dcd5ab1b0ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731435249837 2024-11-12T18:14:13,273 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 198669fc498540c9ad420c4fdeeade92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731435250200 2024-11-12T18:14:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435313270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,280 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc59dd4a50f14186a96ab580d656d0ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731435250579 2024-11-12T18:14:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d71935f9517f4cfaab14c65be4540401 2024-11-12T18:14:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435313321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435313321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435313321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,332 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:13,333 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/52c1c5d5990148d297ead682389e93cd is 50, key is test_row_0/C:col10/1731435250579/Put/seqid=0 2024-11-12T18:14:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435313332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/028970a3f2a64a07849cd22314def633 is 50, key is test_row_0/B:col10/1731435251755/Put/seqid=0 2024-11-12T18:14:13,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741909_1085 (size=13085) 2024-11-12T18:14:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T18:14:13,377 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/52c1c5d5990148d297ead682389e93cd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/52c1c5d5990148d297ead682389e93cd 2024-11-12T18:14:13,389 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 52c1c5d5990148d297ead682389e93cd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:13,389 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:13,390 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435252735; duration=0sec 2024-11-12T18:14:13,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741910_1086 (size=12301) 2024-11-12T18:14:13,390 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:13,390 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:13,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435313590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435313630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435313630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435313641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435313653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:13,797 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/028970a3f2a64a07849cd22314def633 2024-11-12T18:14:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/138927e230b94274aa1417875616fc73 is 50, key is test_row_0/C:col10/1731435251755/Put/seqid=0 2024-11-12T18:14:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741911_1087 (size=12301) 2024-11-12T18:14:13,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/138927e230b94274aa1417875616fc73 2024-11-12T18:14:13,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/d71935f9517f4cfaab14c65be4540401 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d71935f9517f4cfaab14c65be4540401 2024-11-12T18:14:13,883 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d71935f9517f4cfaab14c65be4540401, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T18:14:13,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/028970a3f2a64a07849cd22314def633 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/028970a3f2a64a07849cd22314def633 2024-11-12T18:14:13,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/028970a3f2a64a07849cd22314def633, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T18:14:13,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/138927e230b94274aa1417875616fc73 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/138927e230b94274aa1417875616fc73 2024-11-12T18:14:13,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/138927e230b94274aa1417875616fc73, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T18:14:13,905 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 64a54f5030b48a2f7caee1e78dcb2577 in 1041ms, sequenceid=466, compaction requested=false 2024-11-12T18:14:13,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:13,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:13,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T18:14:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-12T18:14:13,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T18:14:13,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6810 sec 2024-11-12T18:14:13,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6890 sec 2024-11-12T18:14:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:14,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-12T18:14:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:14,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/b6e4146053c64dcdb7a72f078cf95a7a is 50, key is test_row_0/A:col10/1731435252981/Put/seqid=0 2024-11-12T18:14:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741912_1088 (size=14741) 2024-11-12T18:14:14,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435314193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435314201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435314203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435314205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435314205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435314320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435314321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435314321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435314321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T18:14:14,377 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:14,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-12T18:14:14,381 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:14,382 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:14,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T18:14:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T18:14:14,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T18:14:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:14,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435314541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435314541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435314541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:14,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435314544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/b6e4146053c64dcdb7a72f078cf95a7a 2024-11-12T18:14:14,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/57a8267eca1247cbba043da81b953059 is 50, key is test_row_0/B:col10/1731435252981/Put/seqid=0 2024-11-12T18:14:14,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741913_1089 (size=12301) 2024-11-12T18:14:14,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/57a8267eca1247cbba043da81b953059 2024-11-12T18:14:14,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d7df0bf2b612478fae6f7ab2c619b889 is 50, key is test_row_0/C:col10/1731435252981/Put/seqid=0 2024-11-12T18:14:14,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T18:14:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T18:14:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:14,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741914_1090 (size=12301) 2024-11-12T18:14:14,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d7df0bf2b612478fae6f7ab2c619b889 2024-11-12T18:14:14,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/b6e4146053c64dcdb7a72f078cf95a7a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/b6e4146053c64dcdb7a72f078cf95a7a 2024-11-12T18:14:14,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/b6e4146053c64dcdb7a72f078cf95a7a, entries=200, sequenceid=502, filesize=14.4 K 2024-11-12T18:14:14,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/57a8267eca1247cbba043da81b953059 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/57a8267eca1247cbba043da81b953059 2024-11-12T18:14:14,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/57a8267eca1247cbba043da81b953059, entries=150, sequenceid=502, filesize=12.0 K 2024-11-12T18:14:14,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d7df0bf2b612478fae6f7ab2c619b889 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7df0bf2b612478fae6f7ab2c619b889 2024-11-12T18:14:14,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7df0bf2b612478fae6f7ab2c619b889, entries=150, sequenceid=502, filesize=12.0 K 2024-11-12T18:14:14,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=67.09 KB/68700 for 64a54f5030b48a2f7caee1e78dcb2577 in 647ms, sequenceid=502, compaction requested=true 2024-11-12T18:14:14,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:14,769 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:14,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:14,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:14,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:14,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:14,770 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:14,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:14,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:14,770 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:14,770 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:14,770 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,771 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/0c05fbfacb3148d498e9bafce3fbdb54, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d71935f9517f4cfaab14c65be4540401, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/b6e4146053c64dcdb7a72f078cf95a7a] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=39.2 K 2024-11-12T18:14:14,771 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c05fbfacb3148d498e9bafce3fbdb54, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731435250579 2024-11-12T18:14:14,771 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:14,771 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:14,771 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,771 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/364bf0ce7db94aa693dba7e3ff6e1159, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/028970a3f2a64a07849cd22314def633, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/57a8267eca1247cbba043da81b953059] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.8 K 2024-11-12T18:14:14,772 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d71935f9517f4cfaab14c65be4540401, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731435251754 2024-11-12T18:14:14,772 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 364bf0ce7db94aa693dba7e3ff6e1159, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731435250579 2024-11-12T18:14:14,772 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6e4146053c64dcdb7a72f078cf95a7a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731435252980 2024-11-12T18:14:14,788 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 028970a3f2a64a07849cd22314def633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731435251754 2024-11-12T18:14:14,791 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a8267eca1247cbba043da81b953059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731435252981 2024-11-12T18:14:14,819 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:14,820 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/e2978871ea9a42ddaa1b0410e171aed2 is 50, key is test_row_0/A:col10/1731435252981/Put/seqid=0 2024-11-12T18:14:14,830 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#79 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:14,830 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/9bec02162c7e4e8593fe6aa44619c2fd is 50, key is test_row_0/B:col10/1731435252981/Put/seqid=0 2024-11-12T18:14:14,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741915_1091 (size=13187) 2024-11-12T18:14:14,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T18:14:14,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741916_1092 (size=13187) 2024-11-12T18:14:14,855 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:14,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5cf35ae66e604f6596273206158d03dc is 50, key is test_row_0/A:col10/1731435254186/Put/seqid=0 2024-11-12T18:14:14,889 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/9bec02162c7e4e8593fe6aa44619c2fd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9bec02162c7e4e8593fe6aa44619c2fd 2024-11-12T18:14:14,892 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/e2978871ea9a42ddaa1b0410e171aed2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e2978871ea9a42ddaa1b0410e171aed2 2024-11-12T18:14:14,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741917_1093 (size=12301) 2024-11-12T18:14:14,909 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 9bec02162c7e4e8593fe6aa44619c2fd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:14,909 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:14,909 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435254770; duration=0sec 2024-11-12T18:14:14,910 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5cf35ae66e604f6596273206158d03dc 2024-11-12T18:14:14,910 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:14,910 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:14,910 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:14,914 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into e2978871ea9a42ddaa1b0410e171aed2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:14,914 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:14,914 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435254769; duration=0sec 2024-11-12T18:14:14,914 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:14,915 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:14,915 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:14,915 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:14,915 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:14,915 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/52c1c5d5990148d297ead682389e93cd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/138927e230b94274aa1417875616fc73, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7df0bf2b612478fae6f7ab2c619b889] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.8 K 2024-11-12T18:14:14,916 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 52c1c5d5990148d297ead682389e93cd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731435250579 2024-11-12T18:14:14,918 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 138927e230b94274aa1417875616fc73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731435251754 2024-11-12T18:14:14,919 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d7df0bf2b612478fae6f7ab2c619b889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731435252981 2024-11-12T18:14:14,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/67632468b0bb4b4082e7f22026ff683b is 50, key is test_row_0/B:col10/1731435254186/Put/seqid=0 2024-11-12T18:14:14,948 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:14,949 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/565ada2cbca04638b131f8ea6948be91 is 50, key is test_row_0/C:col10/1731435252981/Put/seqid=0 2024-11-12T18:14:14,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435314967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435314967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435314977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435314977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741918_1094 (size=12301) 2024-11-12T18:14:15,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/67632468b0bb4b4082e7f22026ff683b 2024-11-12T18:14:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T18:14:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741919_1095 (size=13187) 2024-11-12T18:14:15,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/a357c3fa361f433fb3efa91b3f442201 is 50, key is test_row_0/C:col10/1731435254186/Put/seqid=0 2024-11-12T18:14:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741920_1096 (size=12301) 2024-11-12T18:14:15,060 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/a357c3fa361f433fb3efa91b3f442201 2024-11-12T18:14:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5cf35ae66e604f6596273206158d03dc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5cf35ae66e604f6596273206158d03dc 2024-11-12T18:14:15,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5cf35ae66e604f6596273206158d03dc, entries=150, sequenceid=515, filesize=12.0 K 2024-11-12T18:14:15,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/67632468b0bb4b4082e7f22026ff683b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67632468b0bb4b4082e7f22026ff683b 2024-11-12T18:14:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435315091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,095 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67632468b0bb4b4082e7f22026ff683b, entries=150, sequenceid=515, filesize=12.0 K 2024-11-12T18:14:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435315091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435315091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435315091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/a357c3fa361f433fb3efa91b3f442201 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/a357c3fa361f433fb3efa91b3f442201 2024-11-12T18:14:15,110 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/a357c3fa361f433fb3efa91b3f442201, entries=150, sequenceid=515, filesize=12.0 K 2024-11-12T18:14:15,112 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 64a54f5030b48a2f7caee1e78dcb2577 in 258ms, sequenceid=515, compaction requested=false 2024-11-12T18:14:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-12T18:14:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-12T18:14:15,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T18:14:15,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 733 msec 2024-11-12T18:14:15,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 745 msec 2024-11-12T18:14:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:15,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:14:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/406355620b8444a28ce76a55ca6bc4a2 is 50, key is test_row_0/A:col10/1731435255220/Put/seqid=0 2024-11-12T18:14:15,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741921_1097 (size=14741) 2024-11-12T18:14:15,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/406355620b8444a28ce76a55ca6bc4a2 2024-11-12T18:14:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435315293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435315301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435315301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/387a9edb932c4ae193c12ef9e753318e is 50, key is test_row_0/B:col10/1731435255220/Put/seqid=0 2024-11-12T18:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435315313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435315313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741922_1098 (size=12301) 2024-11-12T18:14:15,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/387a9edb932c4ae193c12ef9e753318e 2024-11-12T18:14:15,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/1362b7479ff749bcb712c91104f9cc2d is 50, key is test_row_0/C:col10/1731435255220/Put/seqid=0 2024-11-12T18:14:15,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435315410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741923_1099 (size=12301) 2024-11-12T18:14:15,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/1362b7479ff749bcb712c91104f9cc2d 2024-11-12T18:14:15,441 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/565ada2cbca04638b131f8ea6948be91 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/565ada2cbca04638b131f8ea6948be91 2024-11-12T18:14:15,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/406355620b8444a28ce76a55ca6bc4a2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/406355620b8444a28ce76a55ca6bc4a2 2024-11-12T18:14:15,456 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 565ada2cbca04638b131f8ea6948be91(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:15,456 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,456 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435254770; duration=0sec 2024-11-12T18:14:15,456 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:15,456 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:15,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/406355620b8444a28ce76a55ca6bc4a2, entries=200, sequenceid=537, filesize=14.4 K 2024-11-12T18:14:15,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/387a9edb932c4ae193c12ef9e753318e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/387a9edb932c4ae193c12ef9e753318e 2024-11-12T18:14:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/387a9edb932c4ae193c12ef9e753318e, entries=150, sequenceid=537, filesize=12.0 K 2024-11-12T18:14:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/1362b7479ff749bcb712c91104f9cc2d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1362b7479ff749bcb712c91104f9cc2d 2024-11-12T18:14:15,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1362b7479ff749bcb712c91104f9cc2d, entries=150, sequenceid=537, filesize=12.0 K 2024-11-12T18:14:15,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 64a54f5030b48a2f7caee1e78dcb2577 in 263ms, sequenceid=537, compaction requested=true 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,488 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:14:15,488 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:15,489 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:15,489 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:15,489 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,490 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e2978871ea9a42ddaa1b0410e171aed2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5cf35ae66e604f6596273206158d03dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/406355620b8444a28ce76a55ca6bc4a2] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=39.3 K 2024-11-12T18:14:15,490 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:15,490 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2978871ea9a42ddaa1b0410e171aed2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731435252981 2024-11-12T18:14:15,490 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:15,490 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,490 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9bec02162c7e4e8593fe6aa44619c2fd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67632468b0bb4b4082e7f22026ff683b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/387a9edb932c4ae193c12ef9e753318e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.9 K 2024-11-12T18:14:15,490 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf35ae66e604f6596273206158d03dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731435254186 2024-11-12T18:14:15,491 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bec02162c7e4e8593fe6aa44619c2fd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731435252981 2024-11-12T18:14:15,491 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406355620b8444a28ce76a55ca6bc4a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731435254923 2024-11-12T18:14:15,491 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 67632468b0bb4b4082e7f22026ff683b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731435254186 2024-11-12T18:14:15,493 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 387a9edb932c4ae193c12ef9e753318e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731435254923 2024-11-12T18:14:15,508 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:15,509 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/6eceef06684349428487873b6488b4dc is 50, key is test_row_0/A:col10/1731435255220/Put/seqid=0 2024-11-12T18:14:15,513 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:15,514 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/bd26328a6bbd4451a1866994e3d59f96 is 50, key is test_row_0/B:col10/1731435255220/Put/seqid=0 2024-11-12T18:14:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T18:14:15,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:15,521 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-12T18:14:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:15,523 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:15,524 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:15,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:15,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741925_1101 (size=13289) 2024-11-12T18:14:15,551 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/bd26328a6bbd4451a1866994e3d59f96 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/bd26328a6bbd4451a1866994e3d59f96 2024-11-12T18:14:15,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741924_1100 (size=13289) 2024-11-12T18:14:15,571 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/6eceef06684349428487873b6488b4dc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/6eceef06684349428487873b6488b4dc 2024-11-12T18:14:15,575 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into bd26328a6bbd4451a1866994e3d59f96(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:15,575 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,575 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435255488; duration=0sec 2024-11-12T18:14:15,575 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:15,576 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:15,576 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:15,581 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 6eceef06684349428487873b6488b4dc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:15,581 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,582 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435255488; duration=0sec 2024-11-12T18:14:15,582 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:15,582 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:15,589 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:15,590 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:15,590 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,590 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/565ada2cbca04638b131f8ea6948be91, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/a357c3fa361f433fb3efa91b3f442201, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1362b7479ff749bcb712c91104f9cc2d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=36.9 K 2024-11-12T18:14:15,590 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 565ada2cbca04638b131f8ea6948be91, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1731435252981 2024-11-12T18:14:15,591 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a357c3fa361f433fb3efa91b3f442201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1731435254186 2024-11-12T18:14:15,591 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1362b7479ff749bcb712c91104f9cc2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731435254923 2024-11-12T18:14:15,603 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:15,604 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/e412e23409b6473eba806ff7d9c30472 is 50, key is test_row_0/C:col10/1731435255220/Put/seqid=0 2024-11-12T18:14:15,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:15,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741926_1102 (size=13289) 2024-11-12T18:14:15,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/2840eb5feded4877a11f6b769c84efae is 50, key is test_row_0/A:col10/1731435255289/Put/seqid=0 2024-11-12T18:14:15,657 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/e412e23409b6473eba806ff7d9c30472 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e412e23409b6473eba806ff7d9c30472 2024-11-12T18:14:15,671 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into e412e23409b6473eba806ff7d9c30472(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:15,671 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,671 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435255488; duration=0sec 2024-11-12T18:14:15,671 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:15,671 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:15,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435315658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435315665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435315666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435315668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:15,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435315674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741927_1103 (size=17181) 2024-11-12T18:14:15,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/2840eb5feded4877a11f6b769c84efae 2024-11-12T18:14:15,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/c703b6ef572b4243a6e5f010e526a89d is 50, key is test_row_0/B:col10/1731435255289/Put/seqid=0 2024-11-12T18:14:15,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435315781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435315781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435315781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741928_1104 (size=12301) 2024-11-12T18:14:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/c703b6ef572b4243a6e5f010e526a89d 2024-11-12T18:14:15,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435315791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435315791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:15,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/ac166ade286e4ec58b0a4caf5722385a is 50, key is test_row_0/C:col10/1731435255289/Put/seqid=0 2024-11-12T18:14:15,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741929_1105 (size=12301) 2024-11-12T18:14:15,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:15,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:15,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:15,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/ac166ade286e4ec58b0a4caf5722385a 2024-11-12T18:14:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/2840eb5feded4877a11f6b769c84efae as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/2840eb5feded4877a11f6b769c84efae 2024-11-12T18:14:15,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/2840eb5feded4877a11f6b769c84efae, entries=250, sequenceid=554, filesize=16.8 K 2024-11-12T18:14:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/c703b6ef572b4243a6e5f010e526a89d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/c703b6ef572b4243a6e5f010e526a89d 2024-11-12T18:14:15,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/c703b6ef572b4243a6e5f010e526a89d, entries=150, sequenceid=554, filesize=12.0 K 2024-11-12T18:14:15,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/ac166ade286e4ec58b0a4caf5722385a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/ac166ade286e4ec58b0a4caf5722385a 2024-11-12T18:14:15,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/ac166ade286e4ec58b0a4caf5722385a, entries=150, sequenceid=554, filesize=12.0 K 2024-11-12T18:14:15,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 64a54f5030b48a2f7caee1e78dcb2577 in 280ms, sequenceid=554, compaction requested=false 2024-11-12T18:14:15,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:15,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:14:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:15,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:15,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:15,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/f7ecc49b90ea404c9c864b4d667c91bf is 50, key is test_row_0/A:col10/1731435255657/Put/seqid=0 2024-11-12T18:14:16,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435316023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435316026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435316028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435316030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435316035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741930_1106 (size=14741) 2024-11-12T18:14:16,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435316141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435316141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435316141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435316141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435316151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:16,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435316350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435316351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435316351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435316361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435316361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/f7ecc49b90ea404c9c864b4d667c91bf 2024-11-12T18:14:16,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:16,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/eab9409bece84db6879dc633f56de6c8 is 50, key is test_row_0/B:col10/1731435255657/Put/seqid=0 2024-11-12T18:14:16,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741931_1107 (size=12301) 2024-11-12T18:14:16,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:16,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435316660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435316660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435316661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435316671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:16,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/eab9409bece84db6879dc633f56de6c8 2024-11-12T18:14:16,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d7e137d44590420b8358a358c73d36e8 is 50, key is test_row_0/C:col10/1731435255657/Put/seqid=0 2024-11-12T18:14:16,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:16,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741932_1108 (size=12301) 2024-11-12T18:14:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:16,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d7e137d44590420b8358a358c73d36e8 2024-11-12T18:14:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/f7ecc49b90ea404c9c864b4d667c91bf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f7ecc49b90ea404c9c864b4d667c91bf 2024-11-12T18:14:17,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f7ecc49b90ea404c9c864b4d667c91bf, entries=200, sequenceid=574, filesize=14.4 K 2024-11-12T18:14:17,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/eab9409bece84db6879dc633f56de6c8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/eab9409bece84db6879dc633f56de6c8 2024-11-12T18:14:17,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/eab9409bece84db6879dc633f56de6c8, entries=150, sequenceid=574, filesize=12.0 K 2024-11-12T18:14:17,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/d7e137d44590420b8358a358c73d36e8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7e137d44590420b8358a358c73d36e8 2024-11-12T18:14:17,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7e137d44590420b8358a358c73d36e8, entries=150, sequenceid=574, filesize=12.0 K 2024-11-12T18:14:17,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 64a54f5030b48a2f7caee1e78dcb2577 in 1049ms, sequenceid=574, compaction requested=true 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:17,045 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:14:17,045 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:17,047 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:17,047 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:17,047 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:17,047 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/6eceef06684349428487873b6488b4dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/2840eb5feded4877a11f6b769c84efae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f7ecc49b90ea404c9c864b4d667c91bf] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=44.2 K 2024-11-12T18:14:17,048 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:17,048 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:17,048 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:17,048 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/bd26328a6bbd4451a1866994e3d59f96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/c703b6ef572b4243a6e5f010e526a89d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/eab9409bece84db6879dc633f56de6c8] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=37.0 K 2024-11-12T18:14:17,048 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eceef06684349428487873b6488b4dc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731435254923 2024-11-12T18:14:17,053 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2840eb5feded4877a11f6b769c84efae, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731435255228 2024-11-12T18:14:17,053 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting bd26328a6bbd4451a1866994e3d59f96, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731435254923 2024-11-12T18:14:17,053 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ecc49b90ea404c9c864b4d667c91bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731435255641 2024-11-12T18:14:17,053 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c703b6ef572b4243a6e5f010e526a89d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731435255289 2024-11-12T18:14:17,054 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting eab9409bece84db6879dc633f56de6c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731435255657 2024-11-12T18:14:17,084 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:17,085 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/785ec863dec343c4a6698adbc587d529 is 50, key is test_row_0/B:col10/1731435255657/Put/seqid=0 2024-11-12T18:14:17,093 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:17,094 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/73b02c9758894c9b942dd2bf069a1547 is 50, key is test_row_0/A:col10/1731435255657/Put/seqid=0 2024-11-12T18:14:17,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741933_1109 (size=13391) 2024-11-12T18:14:17,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T18:14:17,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:17,144 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:17,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5a5faa8687ee447795e3cc8867b10c00 is 50, key is test_row_0/A:col10/1731435256028/Put/seqid=0 2024-11-12T18:14:17,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741934_1110 (size=13391) 2024-11-12T18:14:17,173 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/785ec863dec343c4a6698adbc587d529 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/785ec863dec343c4a6698adbc587d529 2024-11-12T18:14:17,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. as already flushing 2024-11-12T18:14:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:17,184 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 785ec863dec343c4a6698adbc587d529(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:17,185 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:17,185 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435257045; duration=0sec 2024-11-12T18:14:17,185 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:17,185 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:17,185 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:17,186 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:17,187 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:17,187 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:17,187 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e412e23409b6473eba806ff7d9c30472, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/ac166ade286e4ec58b0a4caf5722385a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7e137d44590420b8358a358c73d36e8] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=37.0 K 2024-11-12T18:14:17,188 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e412e23409b6473eba806ff7d9c30472, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1731435254923 2024-11-12T18:14:17,190 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ac166ade286e4ec58b0a4caf5722385a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1731435255289 2024-11-12T18:14:17,190 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e137d44590420b8358a358c73d36e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731435255657 2024-11-12T18:14:17,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741935_1111 (size=12301) 2024-11-12T18:14:17,197 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5a5faa8687ee447795e3cc8867b10c00 2024-11-12T18:14:17,205 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:17,206 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/dafbb9e3066c4c4bb5b8b75018ead62b is 50, key is test_row_0/C:col10/1731435255657/Put/seqid=0 2024-11-12T18:14:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/30585fcd9ca7498689e4aac8202fbd5e is 50, key is test_row_0/B:col10/1731435256028/Put/seqid=0 2024-11-12T18:14:17,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741936_1112 (size=13391) 2024-11-12T18:14:17,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435317221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741937_1113 (size=12301) 2024-11-12T18:14:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435317227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435317231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435317235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435317238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,263 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/dafbb9e3066c4c4bb5b8b75018ead62b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/dafbb9e3066c4c4bb5b8b75018ead62b 2024-11-12T18:14:17,272 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into dafbb9e3066c4c4bb5b8b75018ead62b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:17,272 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:17,273 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435257045; duration=0sec 2024-11-12T18:14:17,273 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:17,273 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:17,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435317343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435317353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435317353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435317353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435317354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435317560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,568 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/73b02c9758894c9b942dd2bf069a1547 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/73b02c9758894c9b942dd2bf069a1547 2024-11-12T18:14:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435317571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435317572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,576 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 73b02c9758894c9b942dd2bf069a1547(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435317573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,576 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,576 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435257045; duration=0sec 2024-11-12T18:14:17,577 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,577 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435317571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,644 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/30585fcd9ca7498689e4aac8202fbd5e 2024-11-12T18:14:17,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f94ac4d888394119b60f256111d122d7 is 50, key is test_row_0/C:col10/1731435256028/Put/seqid=0 2024-11-12T18:14:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741938_1114 (size=12301) 2024-11-12T18:14:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42566 deadline: 1731435317881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1731435317881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42592 deadline: 1731435317881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42538 deadline: 1731435317881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42574 deadline: 1731435317890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:18,101 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f94ac4d888394119b60f256111d122d7 2024-11-12T18:14:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5a5faa8687ee447795e3cc8867b10c00 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a5faa8687ee447795e3cc8867b10c00 2024-11-12T18:14:18,123 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a5faa8687ee447795e3cc8867b10c00, entries=150, sequenceid=587, filesize=12.0 K 2024-11-12T18:14:18,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/30585fcd9ca7498689e4aac8202fbd5e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30585fcd9ca7498689e4aac8202fbd5e 2024-11-12T18:14:18,138 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30585fcd9ca7498689e4aac8202fbd5e, entries=150, sequenceid=587, filesize=12.0 K 2024-11-12T18:14:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/f94ac4d888394119b60f256111d122d7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f94ac4d888394119b60f256111d122d7 2024-11-12T18:14:18,161 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f94ac4d888394119b60f256111d122d7, entries=150, sequenceid=587, filesize=12.0 K 2024-11-12T18:14:18,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 64a54f5030b48a2f7caee1e78dcb2577 in 1018ms, sequenceid=587, compaction requested=false 2024-11-12T18:14:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:18,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-12T18:14:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-12T18:14:18,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T18:14:18,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6390 sec 2024-11-12T18:14:18,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.6470 sec 2024-11-12T18:14:18,262 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T18:14:18,262 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,263 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T18:14:18,264 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,267 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T18:14:18,267 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,267 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T18:14:18,267 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,274 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,275 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,277 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,277 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,284 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,284 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,285 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,285 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:18,404 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T18:14:18,404 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,404 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,404 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,404 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T18:14:18,404 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,405 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,405 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435257214, max timestamp=1731435258397Num uniques 859; 2024-11-12T18:14:18,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T18:14:18,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64a54f5030b48a2f7caee1e78dcb2577 3/3 column families, dataSize=89.06 KB heapSize=261.20 KB 2024-11-12T18:14:18,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=A 2024-11-12T18:14:18,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:18,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=B 2024-11-12T18:14:18,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:18,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64a54f5030b48a2f7caee1e78dcb2577, store=C 2024-11-12T18:14:18,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:18,409 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T18:14:18,409 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,409 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,409 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T18:14:18,409 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,409 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,409 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,410 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,410 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T18:14:18,411 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:18,411 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,411 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:18,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:18,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5a95958c4a6f491696617b9ee2045ebd is 50, key is test_row_0/A:col10/1731435258404/Put/seqid=0 2024-11-12T18:14:18,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741939_1115 (size=12301) 2024-11-12T18:14:18,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.26 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5a95958c4a6f491696617b9ee2045ebd 2024-11-12T18:14:18,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e0e8eab974024d7aacbeb489260c13ab is 50, key is test_row_0/B:col10/1731435258404/Put/seqid=0 2024-11-12T18:14:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741940_1116 (size=12301) 2024-11-12T18:14:18,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e0e8eab974024d7aacbeb489260c13ab 2024-11-12T18:14:18,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/5ef25ee1e62d4a698ddd07f3f6b299f3 is 50, key is test_row_0/C:col10/1731435258404/Put/seqid=0 2024-11-12T18:14:18,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741941_1117 (size=12301) 2024-11-12T18:14:19,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/5ef25ee1e62d4a698ddd07f3f6b299f3 2024-11-12T18:14:19,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/5a95958c4a6f491696617b9ee2045ebd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a95958c4a6f491696617b9ee2045ebd 2024-11-12T18:14:19,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a95958c4a6f491696617b9ee2045ebd, entries=150, sequenceid=614, filesize=12.0 K 2024-11-12T18:14:19,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/e0e8eab974024d7aacbeb489260c13ab as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e8eab974024d7aacbeb489260c13ab 2024-11-12T18:14:19,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e8eab974024d7aacbeb489260c13ab, entries=150, sequenceid=614, filesize=12.0 K 2024-11-12T18:14:19,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/5ef25ee1e62d4a698ddd07f3f6b299f3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5ef25ee1e62d4a698ddd07f3f6b299f3 2024-11-12T18:14:19,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5ef25ee1e62d4a698ddd07f3f6b299f3, entries=150, sequenceid=614, filesize=12.0 K 2024-11-12T18:14:19,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.19 KB/111806, heapSize ~320.91 KB/328616, currentSize=0 B/0 for 64a54f5030b48a2f7caee1e78dcb2577 in 1045ms, sequenceid=614, compaction requested=true 2024-11-12T18:14:19,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:19,452 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:19,452 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64a54f5030b48a2f7caee1e78dcb2577:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:19,454 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:19,454 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/A is initiating minor compaction (all files) 2024-11-12T18:14:19,454 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/A in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,454 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/73b02c9758894c9b942dd2bf069a1547, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a5faa8687ee447795e3cc8867b10c00, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a95958c4a6f491696617b9ee2045ebd] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=37.1 K 2024-11-12T18:14:19,455 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:19,455 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/B is initiating minor compaction (all files) 2024-11-12T18:14:19,455 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/B in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,455 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/785ec863dec343c4a6698adbc587d529, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30585fcd9ca7498689e4aac8202fbd5e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e8eab974024d7aacbeb489260c13ab] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=37.1 K 2024-11-12T18:14:19,455 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73b02c9758894c9b942dd2bf069a1547, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731435255657 2024-11-12T18:14:19,456 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 785ec863dec343c4a6698adbc587d529, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731435255657 2024-11-12T18:14:19,456 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a5faa8687ee447795e3cc8867b10c00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731435256022 2024-11-12T18:14:19,456 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 30585fcd9ca7498689e4aac8202fbd5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731435256022 2024-11-12T18:14:19,456 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a95958c4a6f491696617b9ee2045ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1731435258403 2024-11-12T18:14:19,457 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e8eab974024d7aacbeb489260c13ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1731435258403 2024-11-12T18:14:19,479 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:19,480 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/9c152048de8f4f2eac0dd64ec4012ce4 is 50, key is test_row_0/B:col10/1731435258404/Put/seqid=0 2024-11-12T18:14:19,491 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:19,492 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7bfc4c627fa1426682dad54c4025a2d1 is 50, key is test_row_0/A:col10/1731435258404/Put/seqid=0 2024-11-12T18:14:19,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741942_1118 (size=13493) 2024-11-12T18:14:19,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741943_1119 (size=13493) 2024-11-12T18:14:19,517 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/B/9c152048de8f4f2eac0dd64ec4012ce4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9c152048de8f4f2eac0dd64ec4012ce4 2024-11-12T18:14:19,526 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/B of 64a54f5030b48a2f7caee1e78dcb2577 into 9c152048de8f4f2eac0dd64ec4012ce4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:19,526 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:19,526 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/B, priority=13, startTime=1731435259452; duration=0sec 2024-11-12T18:14:19,526 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:19,526 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:B 2024-11-12T18:14:19,526 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:19,527 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:19,527 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 64a54f5030b48a2f7caee1e78dcb2577/C is initiating minor compaction (all files) 2024-11-12T18:14:19,527 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64a54f5030b48a2f7caee1e78dcb2577/C in TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,527 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/dafbb9e3066c4c4bb5b8b75018ead62b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f94ac4d888394119b60f256111d122d7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5ef25ee1e62d4a698ddd07f3f6b299f3] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp, totalSize=37.1 K 2024-11-12T18:14:19,528 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting dafbb9e3066c4c4bb5b8b75018ead62b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731435255657 2024-11-12T18:14:19,528 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f94ac4d888394119b60f256111d122d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1731435256022 2024-11-12T18:14:19,529 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef25ee1e62d4a698ddd07f3f6b299f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1731435258403 2024-11-12T18:14:19,538 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64a54f5030b48a2f7caee1e78dcb2577#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:19,539 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/5305b64a6cb2422ea53d6a050e3fca09 is 50, key is test_row_0/C:col10/1731435258404/Put/seqid=0 2024-11-12T18:14:19,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741944_1120 (size=13493) 2024-11-12T18:14:19,557 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/C/5305b64a6cb2422ea53d6a050e3fca09 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5305b64a6cb2422ea53d6a050e3fca09 2024-11-12T18:14:19,566 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/C of 64a54f5030b48a2f7caee1e78dcb2577 into 5305b64a6cb2422ea53d6a050e3fca09(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:19,566 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:19,566 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/C, priority=13, startTime=1731435259452; duration=0sec 2024-11-12T18:14:19,566 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:19,566 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:C 2024-11-12T18:14:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T18:14:19,681 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:19,681 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1954 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1892 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 908 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2724 rows 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 914 2024-11-12T18:14:19,682 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2734 rows 2024-11-12T18:14:19,682 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:14:19,682 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:19,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:19,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:19,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:19,689 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T18:14:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T18:14:19,697 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435259697"}]},"ts":"1731435259697"} 2024-11-12T18:14:19,700 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T18:14:19,700 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T18:14:19,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:14:19,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, UNASSIGN}] 2024-11-12T18:14:19,709 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, UNASSIGN 2024-11-12T18:14:19,711 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=64a54f5030b48a2f7caee1e78dcb2577, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:19,712 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=9911683f163c,43757,1731435233300, table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:14:19,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, UNASSIGN because future has completed 2024-11-12T18:14:19,714 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:14:19,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:14:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T18:14:19,874 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:19,874 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:14:19,875 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 64a54f5030b48a2f7caee1e78dcb2577, disabling compactions & flushes 2024-11-12T18:14:19,875 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,917 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/.tmp/A/7bfc4c627fa1426682dad54c4025a2d1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7bfc4c627fa1426682dad54c4025a2d1 2024-11-12T18:14:19,924 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64a54f5030b48a2f7caee1e78dcb2577/A of 64a54f5030b48a2f7caee1e78dcb2577 into 7bfc4c627fa1426682dad54c4025a2d1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:19,924 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64a54f5030b48a2f7caee1e78dcb2577: 2024-11-12T18:14:19,924 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,924 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577., storeName=64a54f5030b48a2f7caee1e78dcb2577/A, priority=13, startTime=1731435259451; duration=0sec 2024-11-12T18:14:19,924 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,924 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. after waiting 0 ms 2024-11-12T18:14:19,924 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:19,924 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:19,924 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64a54f5030b48a2f7caee1e78dcb2577:A 2024-11-12T18:14:19,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7d0fee488e794bfb8105cc765c3404a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/14775c1f8051482fa92e609c617ad1b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/8e0b6a49d7fc40fca6f0c79ead0293b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f1115d2d251a41dfa8bb63324d0a2dcd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/35f88f3867364cf681422395a0d200a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e6379db56bf4d98bfc8c46472a5a884, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/40f4ad51956e4043b082a465219db6ae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/64c3eb733c4e42c69ff471672c98732f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/84ad10c6001248078201f7033dc180fd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d0fe28631b3547efaa90353ddbb79114, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fed7ee360dc043179e07e79cf63cf5eb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/594ae7c967eb48cdad829a8f9d10f018, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/272bc79ebcc742d9ae0c1b52e6165bff, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/9a676def3c1f4146981c3bf4b22904b8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/925fee6d86e6481286ab654a0dc69e34, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e09c8943c5d44acc8138e3dc972d480b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/623af9c99ed740d78fb493bc09de0272, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fc4ddb818baa47f0b5a4d43a33eafda4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/356483b6b7e647c6a7429869c01e8555, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/44f0bd788bd34f9ab239b72b7e646d74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e61dbde5a1a44c4887b29b11d1b52b7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5487c8b036f946c499884c26f95125e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d22b528f440746c59e3439d440c8ee38, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/0c05fbfacb3148d498e9bafce3fbdb54, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d71935f9517f4cfaab14c65be4540401, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/b6e4146053c64dcdb7a72f078cf95a7a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e2978871ea9a42ddaa1b0410e171aed2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5cf35ae66e604f6596273206158d03dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/406355620b8444a28ce76a55ca6bc4a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/6eceef06684349428487873b6488b4dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/2840eb5feded4877a11f6b769c84efae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f7ecc49b90ea404c9c864b4d667c91bf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/73b02c9758894c9b942dd2bf069a1547, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a5faa8687ee447795e3cc8867b10c00, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a95958c4a6f491696617b9ee2045ebd] to archive 2024-11-12T18:14:19,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:14:19,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7d0fee488e794bfb8105cc765c3404a2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7d0fee488e794bfb8105cc765c3404a2 2024-11-12T18:14:19,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/14775c1f8051482fa92e609c617ad1b1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/14775c1f8051482fa92e609c617ad1b1 2024-11-12T18:14:19,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/8e0b6a49d7fc40fca6f0c79ead0293b1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/8e0b6a49d7fc40fca6f0c79ead0293b1 2024-11-12T18:14:19,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f1115d2d251a41dfa8bb63324d0a2dcd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f1115d2d251a41dfa8bb63324d0a2dcd 2024-11-12T18:14:19,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/35f88f3867364cf681422395a0d200a2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/35f88f3867364cf681422395a0d200a2 2024-11-12T18:14:19,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e6379db56bf4d98bfc8c46472a5a884 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e6379db56bf4d98bfc8c46472a5a884 2024-11-12T18:14:19,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/40f4ad51956e4043b082a465219db6ae to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/40f4ad51956e4043b082a465219db6ae 2024-11-12T18:14:19,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/64c3eb733c4e42c69ff471672c98732f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/64c3eb733c4e42c69ff471672c98732f 2024-11-12T18:14:19,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/84ad10c6001248078201f7033dc180fd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/84ad10c6001248078201f7033dc180fd 2024-11-12T18:14:19,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d0fe28631b3547efaa90353ddbb79114 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d0fe28631b3547efaa90353ddbb79114 2024-11-12T18:14:19,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fed7ee360dc043179e07e79cf63cf5eb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fed7ee360dc043179e07e79cf63cf5eb 2024-11-12T18:14:19,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/594ae7c967eb48cdad829a8f9d10f018 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/594ae7c967eb48cdad829a8f9d10f018 2024-11-12T18:14:19,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/272bc79ebcc742d9ae0c1b52e6165bff to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/272bc79ebcc742d9ae0c1b52e6165bff 2024-11-12T18:14:19,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/9a676def3c1f4146981c3bf4b22904b8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/9a676def3c1f4146981c3bf4b22904b8 2024-11-12T18:14:19,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/925fee6d86e6481286ab654a0dc69e34 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/925fee6d86e6481286ab654a0dc69e34 2024-11-12T18:14:19,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e09c8943c5d44acc8138e3dc972d480b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e09c8943c5d44acc8138e3dc972d480b 2024-11-12T18:14:19,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/623af9c99ed740d78fb493bc09de0272 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/623af9c99ed740d78fb493bc09de0272 2024-11-12T18:14:19,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fc4ddb818baa47f0b5a4d43a33eafda4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/fc4ddb818baa47f0b5a4d43a33eafda4 2024-11-12T18:14:19,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/356483b6b7e647c6a7429869c01e8555 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/356483b6b7e647c6a7429869c01e8555 2024-11-12T18:14:19,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/44f0bd788bd34f9ab239b72b7e646d74 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/44f0bd788bd34f9ab239b72b7e646d74 2024-11-12T18:14:19,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e61dbde5a1a44c4887b29b11d1b52b7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7e61dbde5a1a44c4887b29b11d1b52b7 2024-11-12T18:14:19,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5487c8b036f946c499884c26f95125e7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5487c8b036f946c499884c26f95125e7 2024-11-12T18:14:19,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d22b528f440746c59e3439d440c8ee38 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d22b528f440746c59e3439d440c8ee38 2024-11-12T18:14:19,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/0c05fbfacb3148d498e9bafce3fbdb54 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/0c05fbfacb3148d498e9bafce3fbdb54 2024-11-12T18:14:19,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d71935f9517f4cfaab14c65be4540401 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/d71935f9517f4cfaab14c65be4540401 2024-11-12T18:14:19,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/b6e4146053c64dcdb7a72f078cf95a7a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/b6e4146053c64dcdb7a72f078cf95a7a 2024-11-12T18:14:19,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e2978871ea9a42ddaa1b0410e171aed2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/e2978871ea9a42ddaa1b0410e171aed2 2024-11-12T18:14:19,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5cf35ae66e604f6596273206158d03dc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5cf35ae66e604f6596273206158d03dc 2024-11-12T18:14:19,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/406355620b8444a28ce76a55ca6bc4a2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/406355620b8444a28ce76a55ca6bc4a2 2024-11-12T18:14:19,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/6eceef06684349428487873b6488b4dc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/6eceef06684349428487873b6488b4dc 2024-11-12T18:14:19,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/2840eb5feded4877a11f6b769c84efae to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/2840eb5feded4877a11f6b769c84efae 2024-11-12T18:14:19,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f7ecc49b90ea404c9c864b4d667c91bf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/f7ecc49b90ea404c9c864b4d667c91bf 2024-11-12T18:14:19,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/73b02c9758894c9b942dd2bf069a1547 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/73b02c9758894c9b942dd2bf069a1547 2024-11-12T18:14:19,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a5faa8687ee447795e3cc8867b10c00 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a5faa8687ee447795e3cc8867b10c00 2024-11-12T18:14:19,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a95958c4a6f491696617b9ee2045ebd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/5a95958c4a6f491696617b9ee2045ebd 2024-11-12T18:14:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T18:14:20,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e4b4de17ffa54271a55849e7f626bb22, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67e3e14bc3f74b2999c6448e76fc1bc2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ca0b10d38f1e40b3a1c21b10e9ebfd21, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b0a0282cbe73436d9f8c8de9fe8d44bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ad509a878214477eacaf69a044836c24, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e16f78dd1c6d4b7aac00f79987dc48d7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aaae18d88c46490e9ebf3edcf0a106ac, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ea8766f7e47b4f3ab849f92b79489ae4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30650d44956e4f8dbd25d8e1f8463d32, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f2c5487c58e34132bac1dcb970b80b33, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b2e89678ba86470ab11996d1af5f1225, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/6ab75664b82c4c3e94c26acacaae184a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/23c372e33c9640fdbb2ab2232759d28c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/813a090b49c64fa2adbb119a1303970d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7ddfa30ef3cb4c439ce7e71770304b58, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f49ee59899e14cc98c961831b7a6f747, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/da307cef80ef43cd853e495d0eee77b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ff563f62b26a45bdb1c69069047e31b0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e2fe7c3d794b1580d8c7681bf69997, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/48499a4021364ba196f7b8cad51c797f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7077663fe45843cca5806d165f1c90b8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/5b729a75cac64eedad70fa9134efeed8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/364bf0ce7db94aa693dba7e3ff6e1159, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aee3e026f8bc4d47abf5f55875ad523b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/028970a3f2a64a07849cd22314def633, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9bec02162c7e4e8593fe6aa44619c2fd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/57a8267eca1247cbba043da81b953059, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67632468b0bb4b4082e7f22026ff683b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/bd26328a6bbd4451a1866994e3d59f96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/387a9edb932c4ae193c12ef9e753318e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/c703b6ef572b4243a6e5f010e526a89d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/785ec863dec343c4a6698adbc587d529, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/eab9409bece84db6879dc633f56de6c8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30585fcd9ca7498689e4aac8202fbd5e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e8eab974024d7aacbeb489260c13ab] to archive 2024-11-12T18:14:20,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:14:20,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e4b4de17ffa54271a55849e7f626bb22 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e4b4de17ffa54271a55849e7f626bb22 2024-11-12T18:14:20,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67e3e14bc3f74b2999c6448e76fc1bc2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67e3e14bc3f74b2999c6448e76fc1bc2 2024-11-12T18:14:20,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ca0b10d38f1e40b3a1c21b10e9ebfd21 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ca0b10d38f1e40b3a1c21b10e9ebfd21 2024-11-12T18:14:20,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b0a0282cbe73436d9f8c8de9fe8d44bb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b0a0282cbe73436d9f8c8de9fe8d44bb 2024-11-12T18:14:20,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ad509a878214477eacaf69a044836c24 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ad509a878214477eacaf69a044836c24 2024-11-12T18:14:20,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e16f78dd1c6d4b7aac00f79987dc48d7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e16f78dd1c6d4b7aac00f79987dc48d7 2024-11-12T18:14:20,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aaae18d88c46490e9ebf3edcf0a106ac to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aaae18d88c46490e9ebf3edcf0a106ac 2024-11-12T18:14:20,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ea8766f7e47b4f3ab849f92b79489ae4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ea8766f7e47b4f3ab849f92b79489ae4 2024-11-12T18:14:20,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30650d44956e4f8dbd25d8e1f8463d32 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30650d44956e4f8dbd25d8e1f8463d32 2024-11-12T18:14:20,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f2c5487c58e34132bac1dcb970b80b33 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f2c5487c58e34132bac1dcb970b80b33 2024-11-12T18:14:20,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b2e89678ba86470ab11996d1af5f1225 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/b2e89678ba86470ab11996d1af5f1225 2024-11-12T18:14:20,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/6ab75664b82c4c3e94c26acacaae184a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/6ab75664b82c4c3e94c26acacaae184a 2024-11-12T18:14:20,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/23c372e33c9640fdbb2ab2232759d28c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/23c372e33c9640fdbb2ab2232759d28c 2024-11-12T18:14:20,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/813a090b49c64fa2adbb119a1303970d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/813a090b49c64fa2adbb119a1303970d 2024-11-12T18:14:20,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7ddfa30ef3cb4c439ce7e71770304b58 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7ddfa30ef3cb4c439ce7e71770304b58 2024-11-12T18:14:20,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f49ee59899e14cc98c961831b7a6f747 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/f49ee59899e14cc98c961831b7a6f747 2024-11-12T18:14:20,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/da307cef80ef43cd853e495d0eee77b1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/da307cef80ef43cd853e495d0eee77b1 2024-11-12T18:14:20,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ff563f62b26a45bdb1c69069047e31b0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/ff563f62b26a45bdb1c69069047e31b0 2024-11-12T18:14:20,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e2fe7c3d794b1580d8c7681bf69997 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e2fe7c3d794b1580d8c7681bf69997 2024-11-12T18:14:20,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/48499a4021364ba196f7b8cad51c797f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/48499a4021364ba196f7b8cad51c797f 2024-11-12T18:14:20,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7077663fe45843cca5806d165f1c90b8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/7077663fe45843cca5806d165f1c90b8 2024-11-12T18:14:20,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/5b729a75cac64eedad70fa9134efeed8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/5b729a75cac64eedad70fa9134efeed8 2024-11-12T18:14:20,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/364bf0ce7db94aa693dba7e3ff6e1159 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/364bf0ce7db94aa693dba7e3ff6e1159 2024-11-12T18:14:20,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aee3e026f8bc4d47abf5f55875ad523b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/aee3e026f8bc4d47abf5f55875ad523b 2024-11-12T18:14:20,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/028970a3f2a64a07849cd22314def633 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/028970a3f2a64a07849cd22314def633 2024-11-12T18:14:20,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9bec02162c7e4e8593fe6aa44619c2fd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9bec02162c7e4e8593fe6aa44619c2fd 2024-11-12T18:14:20,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/57a8267eca1247cbba043da81b953059 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/57a8267eca1247cbba043da81b953059 2024-11-12T18:14:20,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67632468b0bb4b4082e7f22026ff683b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/67632468b0bb4b4082e7f22026ff683b 2024-11-12T18:14:20,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/bd26328a6bbd4451a1866994e3d59f96 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/bd26328a6bbd4451a1866994e3d59f96 2024-11-12T18:14:20,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/387a9edb932c4ae193c12ef9e753318e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/387a9edb932c4ae193c12ef9e753318e 2024-11-12T18:14:20,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/c703b6ef572b4243a6e5f010e526a89d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/c703b6ef572b4243a6e5f010e526a89d 2024-11-12T18:14:20,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/785ec863dec343c4a6698adbc587d529 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/785ec863dec343c4a6698adbc587d529 2024-11-12T18:14:20,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/eab9409bece84db6879dc633f56de6c8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/eab9409bece84db6879dc633f56de6c8 2024-11-12T18:14:20,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30585fcd9ca7498689e4aac8202fbd5e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/30585fcd9ca7498689e4aac8202fbd5e 2024-11-12T18:14:20,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e8eab974024d7aacbeb489260c13ab to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/e0e8eab974024d7aacbeb489260c13ab 2024-11-12T18:14:20,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/aa3e1b84d5e649f8b51ec88715692af3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/751bd946fa2340acaa85859ae495ce1d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/3ddb951da62a4db697a271c9945d5380, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f414852a274241e7854bb74d3d3cc251, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f6b5283c57db4f61891f3df784b5a94f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d5c307c4763f45339a730b159d6c5d7a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c96044136be2407f9116e0b4769419d0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1ee71b841ded4400aab911aa62af5dfd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/29769615ea8f415b8d2743d286926fc8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/085ae8ffc106468880dd1eda47f50460, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc58549414a041f8b6fe5340d17d78a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/251a968a6e864a6a93eba2ad26f7442e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/4560412e524248c9911b35d037c4ee1e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c2b95dab00b04547ba6b4e613ef5a12c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/8bc1112dbc9e44cc9a44b9592a91a83c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/b1848a3627934656a0385602de0df341, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/30d5cd9e138b4d659ac9c84660ef7ca2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/97a4553c6f214b1d92ee6092112bd04e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7644c3ffcd4a452b9938b42e40315f85, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e566589145d54246b1f23dcd5ab1b0ec, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7319ebbdeeb44a789f3f5cc9c10764c1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/198669fc498540c9ad420c4fdeeade92, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/52c1c5d5990148d297ead682389e93cd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc59dd4a50f14186a96ab580d656d0ba, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/138927e230b94274aa1417875616fc73, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/565ada2cbca04638b131f8ea6948be91, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7df0bf2b612478fae6f7ab2c619b889, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/a357c3fa361f433fb3efa91b3f442201, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e412e23409b6473eba806ff7d9c30472, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1362b7479ff749bcb712c91104f9cc2d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/ac166ade286e4ec58b0a4caf5722385a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/dafbb9e3066c4c4bb5b8b75018ead62b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7e137d44590420b8358a358c73d36e8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f94ac4d888394119b60f256111d122d7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5ef25ee1e62d4a698ddd07f3f6b299f3] to archive 2024-11-12T18:14:20,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:14:20,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/aa3e1b84d5e649f8b51ec88715692af3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/aa3e1b84d5e649f8b51ec88715692af3 2024-11-12T18:14:20,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/751bd946fa2340acaa85859ae495ce1d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/751bd946fa2340acaa85859ae495ce1d 2024-11-12T18:14:20,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/3ddb951da62a4db697a271c9945d5380 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/3ddb951da62a4db697a271c9945d5380 2024-11-12T18:14:20,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f414852a274241e7854bb74d3d3cc251 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f414852a274241e7854bb74d3d3cc251 2024-11-12T18:14:20,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f6b5283c57db4f61891f3df784b5a94f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f6b5283c57db4f61891f3df784b5a94f 2024-11-12T18:14:20,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d5c307c4763f45339a730b159d6c5d7a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d5c307c4763f45339a730b159d6c5d7a 2024-11-12T18:14:20,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c96044136be2407f9116e0b4769419d0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c96044136be2407f9116e0b4769419d0 2024-11-12T18:14:20,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1ee71b841ded4400aab911aa62af5dfd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1ee71b841ded4400aab911aa62af5dfd 2024-11-12T18:14:20,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/29769615ea8f415b8d2743d286926fc8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/29769615ea8f415b8d2743d286926fc8 2024-11-12T18:14:20,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/085ae8ffc106468880dd1eda47f50460 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/085ae8ffc106468880dd1eda47f50460 2024-11-12T18:14:20,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc58549414a041f8b6fe5340d17d78a2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc58549414a041f8b6fe5340d17d78a2 2024-11-12T18:14:20,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/251a968a6e864a6a93eba2ad26f7442e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/251a968a6e864a6a93eba2ad26f7442e 2024-11-12T18:14:20,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/4560412e524248c9911b35d037c4ee1e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/4560412e524248c9911b35d037c4ee1e 2024-11-12T18:14:20,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c2b95dab00b04547ba6b4e613ef5a12c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/c2b95dab00b04547ba6b4e613ef5a12c 2024-11-12T18:14:20,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/8bc1112dbc9e44cc9a44b9592a91a83c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/8bc1112dbc9e44cc9a44b9592a91a83c 2024-11-12T18:14:20,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/b1848a3627934656a0385602de0df341 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/b1848a3627934656a0385602de0df341 2024-11-12T18:14:20,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/30d5cd9e138b4d659ac9c84660ef7ca2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/30d5cd9e138b4d659ac9c84660ef7ca2 2024-11-12T18:14:20,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/97a4553c6f214b1d92ee6092112bd04e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/97a4553c6f214b1d92ee6092112bd04e 2024-11-12T18:14:20,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7644c3ffcd4a452b9938b42e40315f85 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7644c3ffcd4a452b9938b42e40315f85 2024-11-12T18:14:20,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e566589145d54246b1f23dcd5ab1b0ec to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e566589145d54246b1f23dcd5ab1b0ec 2024-11-12T18:14:20,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7319ebbdeeb44a789f3f5cc9c10764c1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/7319ebbdeeb44a789f3f5cc9c10764c1 2024-11-12T18:14:20,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/198669fc498540c9ad420c4fdeeade92 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/198669fc498540c9ad420c4fdeeade92 2024-11-12T18:14:20,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/52c1c5d5990148d297ead682389e93cd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/52c1c5d5990148d297ead682389e93cd 2024-11-12T18:14:20,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc59dd4a50f14186a96ab580d656d0ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/cc59dd4a50f14186a96ab580d656d0ba 2024-11-12T18:14:20,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/138927e230b94274aa1417875616fc73 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/138927e230b94274aa1417875616fc73 2024-11-12T18:14:20,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/565ada2cbca04638b131f8ea6948be91 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/565ada2cbca04638b131f8ea6948be91 2024-11-12T18:14:20,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7df0bf2b612478fae6f7ab2c619b889 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7df0bf2b612478fae6f7ab2c619b889 2024-11-12T18:14:20,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/a357c3fa361f433fb3efa91b3f442201 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/a357c3fa361f433fb3efa91b3f442201 2024-11-12T18:14:20,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e412e23409b6473eba806ff7d9c30472 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/e412e23409b6473eba806ff7d9c30472 2024-11-12T18:14:20,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1362b7479ff749bcb712c91104f9cc2d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/1362b7479ff749bcb712c91104f9cc2d 2024-11-12T18:14:20,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/ac166ade286e4ec58b0a4caf5722385a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/ac166ade286e4ec58b0a4caf5722385a 2024-11-12T18:14:20,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/dafbb9e3066c4c4bb5b8b75018ead62b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/dafbb9e3066c4c4bb5b8b75018ead62b 2024-11-12T18:14:20,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7e137d44590420b8358a358c73d36e8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/d7e137d44590420b8358a358c73d36e8 2024-11-12T18:14:20,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f94ac4d888394119b60f256111d122d7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/f94ac4d888394119b60f256111d122d7 2024-11-12T18:14:20,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5ef25ee1e62d4a698ddd07f3f6b299f3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5ef25ee1e62d4a698ddd07f3f6b299f3 2024-11-12T18:14:20,208 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/recovered.edits/620.seqid, newMaxSeqId=620, maxSeqId=1 2024-11-12T18:14:20,209 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577. 2024-11-12T18:14:20,210 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 64a54f5030b48a2f7caee1e78dcb2577: Waiting for close lock at 1731435259875Running coprocessor pre-close hooks at 1731435259875Disabling compacts and flushes for region at 1731435259875Disabling writes for close at 1731435259924 (+49 ms)Writing region close event to WAL at 1731435260197 (+273 ms)Running coprocessor post-close hooks at 1731435260209 (+12 ms)Closed at 1731435260209 2024-11-12T18:14:20,213 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:20,214 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=64a54f5030b48a2f7caee1e78dcb2577, regionState=CLOSED 2024-11-12T18:14:20,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:14:20,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T18:14:20,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 64a54f5030b48a2f7caee1e78dcb2577, server=9911683f163c,43757,1731435233300 in 514 msec 2024-11-12T18:14:20,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-12T18:14:20,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64a54f5030b48a2f7caee1e78dcb2577, UNASSIGN in 523 msec 2024-11-12T18:14:20,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T18:14:20,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-11-12T18:14:20,239 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435260238"}]},"ts":"1731435260238"} 2024-11-12T18:14:20,240 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T18:14:20,240 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T18:14:20,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 550 msec 2024-11-12T18:14:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T18:14:20,321 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:20,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T18:14:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,333 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,335 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T18:14:20,353 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:20,359 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/recovered.edits] 2024-11-12T18:14:20,367 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7bfc4c627fa1426682dad54c4025a2d1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/A/7bfc4c627fa1426682dad54c4025a2d1 2024-11-12T18:14:20,371 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9c152048de8f4f2eac0dd64ec4012ce4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/B/9c152048de8f4f2eac0dd64ec4012ce4 2024-11-12T18:14:20,389 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5305b64a6cb2422ea53d6a050e3fca09 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/C/5305b64a6cb2422ea53d6a050e3fca09 2024-11-12T18:14:20,409 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/recovered.edits/620.seqid to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577/recovered.edits/620.seqid 2024-11-12T18:14:20,410 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/64a54f5030b48a2f7caee1e78dcb2577 2024-11-12T18:14:20,410 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T18:14:20,426 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T18:14:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T18:14:20,450 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T18:14:20,524 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T18:14:20,526 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,527 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T18:14:20,527 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731435260527"}]},"ts":"9223372036854775807"} 2024-11-12T18:14:20,535 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T18:14:20,535 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 64a54f5030b48a2f7caee1e78dcb2577, NAME => 'TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T18:14:20,535 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T18:14:20,538 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731435260536"}]},"ts":"9223372036854775807"} 2024-11-12T18:14:20,542 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T18:14:20,544 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 217 msec 2024-11-12T18:14:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T18:14:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T18:14:20,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:20,669 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1458389610_22 at /127.0.0.1:49294 [Waiting for operation #322] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9911683f163c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9911683f163c:43757-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1458389610_22 at /127.0.0.1:35728 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2141733817_22 at /127.0.0.1:38678 [Waiting for operation #314] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9911683f163c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=858 (was 703) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2983 (was 3283) 2024-11-12T18:14:20,686 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=858, ProcessCount=11, AvailableMemoryMB=2980 2024-11-12T18:14:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:14:20,688 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:14:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:20,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:14:20,691 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:20,691 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-12T18:14:20,692 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:14:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-12T18:14:20,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741945_1121 (size=1061) 2024-11-12T18:14:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-12T18:14:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-12T18:14:21,106 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:14:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741946_1122 (size=53) 2024-11-12T18:14:21,126 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:21,126 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae79f5068ec1983ef2be0dd82f4139ba, disabling compactions & flushes 2024-11-12T18:14:21,126 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,126 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,126 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. after waiting 0 ms 2024-11-12T18:14:21,126 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,126 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,126 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae79f5068ec1983ef2be0dd82f4139ba: Waiting for close lock at 1731435261126Disabling compacts and flushes for region at 1731435261126Disabling writes for close at 1731435261126Writing region close event to WAL at 1731435261126Closed at 1731435261126 2024-11-12T18:14:21,130 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:14:21,130 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731435261130"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435261130"}]},"ts":"1731435261130"} 2024-11-12T18:14:21,133 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T18:14:21,135 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:14:21,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435261135"}]},"ts":"1731435261135"} 2024-11-12T18:14:21,137 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T18:14:21,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, ASSIGN}] 2024-11-12T18:14:21,139 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, ASSIGN 2024-11-12T18:14:21,140 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, ASSIGN; state=OFFLINE, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=false 2024-11-12T18:14:21,241 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T18:14:21,293 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:21,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, ASSIGN because future has completed 2024-11-12T18:14:21,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:14:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-12T18:14:21,453 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,453 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:14:21,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:21,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,454 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,456 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,457 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:21,458 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae79f5068ec1983ef2be0dd82f4139ba columnFamilyName A 2024-11-12T18:14:21,458 DEBUG [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:21,458 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(327): Store=ae79f5068ec1983ef2be0dd82f4139ba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:21,459 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,460 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:21,460 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae79f5068ec1983ef2be0dd82f4139ba columnFamilyName B 2024-11-12T18:14:21,460 DEBUG [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:21,460 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(327): Store=ae79f5068ec1983ef2be0dd82f4139ba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:21,460 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,462 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:21,462 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae79f5068ec1983ef2be0dd82f4139ba columnFamilyName C 2024-11-12T18:14:21,462 DEBUG [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:21,462 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(327): Store=ae79f5068ec1983ef2be0dd82f4139ba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:21,462 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,463 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,463 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,464 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,465 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,465 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,465 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:14:21,467 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,471 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:21,471 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened ae79f5068ec1983ef2be0dd82f4139ba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58782140, jitterRate=-0.12407785654067993}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:14:21,472 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:21,473 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for ae79f5068ec1983ef2be0dd82f4139ba: Running coprocessor pre-open hook at 1731435261454Writing region info on filesystem at 1731435261454Initializing all the Stores at 1731435261455 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435261455Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435261455Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435261455Cleaning up temporary data from old regions at 1731435261465 (+10 ms)Running coprocessor post-open hooks at 1731435261472 (+7 ms)Region opened successfully at 1731435261473 (+1 ms) 2024-11-12T18:14:21,474 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., pid=32, masterSystemTime=1731435261450 2024-11-12T18:14:21,477 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,477 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:21,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:21,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:14:21,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T18:14:21,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 in 185 msec 2024-11-12T18:14:21,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-12T18:14:21,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, ASSIGN in 346 msec 2024-11-12T18:14:21,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:14:21,487 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435261487"}]},"ts":"1731435261487"} 2024-11-12T18:14:21,488 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T18:14:21,489 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:14:21,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 800 msec 2024-11-12T18:14:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-12T18:14:21,831 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:21,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:21,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:21,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:21,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1d8c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:21,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:21,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121de19e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:21,840 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:21,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:21,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:14:21,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:14:21,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c90b79a 2024-11-12T18:14:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:14:21,848 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:14:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:14:21,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:14:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:21,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741947_1123 (size=1097) 2024-11-12T18:14:22,280 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T18:14:22,280 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T18:14:22,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:14:22,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, REOPEN/MOVE}] 2024-11-12T18:14:22,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, REOPEN/MOVE 2024-11-12T18:14:22,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:22,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, REOPEN/MOVE because future has completed 2024-11-12T18:14:22,295 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:14:22,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:14:22,447 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,447 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:14:22,448 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing ae79f5068ec1983ef2be0dd82f4139ba, disabling compactions & flushes 2024-11-12T18:14:22,448 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,448 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,448 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. after waiting 0 ms 2024-11-12T18:14:22,448 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,452 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:14:22,452 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,452 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for ae79f5068ec1983ef2be0dd82f4139ba: Waiting for close lock at 1731435262448Running coprocessor pre-close hooks at 1731435262448Disabling compacts and flushes for region at 1731435262448Disabling writes for close at 1731435262448Writing region close event to WAL at 1731435262448Running coprocessor post-close hooks at 1731435262452 (+4 ms)Closed at 1731435262452 2024-11-12T18:14:22,453 WARN [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: ae79f5068ec1983ef2be0dd82f4139ba to self. 2024-11-12T18:14:22,455 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,455 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=CLOSED 2024-11-12T18:14:22,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:14:22,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-12T18:14:22,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 in 163 msec 2024-11-12T18:14:22,460 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, REOPEN/MOVE; state=CLOSED, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=true 2024-11-12T18:14:22,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:22,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, REOPEN/MOVE because future has completed 2024-11-12T18:14:22,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:14:22,768 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,768 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:14:22,768 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,768 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:22,768 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,768 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,771 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,772 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:22,777 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae79f5068ec1983ef2be0dd82f4139ba columnFamilyName A 2024-11-12T18:14:22,779 DEBUG [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:22,779 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(327): Store=ae79f5068ec1983ef2be0dd82f4139ba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:22,780 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,781 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:22,781 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae79f5068ec1983ef2be0dd82f4139ba columnFamilyName B 2024-11-12T18:14:22,781 DEBUG [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:22,781 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(327): Store=ae79f5068ec1983ef2be0dd82f4139ba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:22,782 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,782 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:22,783 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae79f5068ec1983ef2be0dd82f4139ba columnFamilyName C 2024-11-12T18:14:22,783 DEBUG [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:22,783 INFO [StoreOpener-ae79f5068ec1983ef2be0dd82f4139ba-1 {}] regionserver.HStore(327): Store=ae79f5068ec1983ef2be0dd82f4139ba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:22,783 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,784 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,784 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,786 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,787 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,787 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,787 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:14:22,789 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,790 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened ae79f5068ec1983ef2be0dd82f4139ba; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74582123, jitterRate=0.11136023700237274}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:14:22,790 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,791 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for ae79f5068ec1983ef2be0dd82f4139ba: Running coprocessor pre-open hook at 1731435262769Writing region info on filesystem at 1731435262769Initializing all the Stores at 1731435262769Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435262769Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435262770 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435262770Cleaning up temporary data from old regions at 1731435262787 (+17 ms)Running coprocessor post-open hooks at 1731435262790 (+3 ms)Region opened successfully at 1731435262790 2024-11-12T18:14:22,791 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., pid=37, masterSystemTime=1731435262764 2024-11-12T18:14:22,793 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,793 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:22,794 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=OPEN, openSeqNum=5, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:22,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:14:22,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-12T18:14:22,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 in 184 msec 2024-11-12T18:14:22,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T18:14:22,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, REOPEN/MOVE in 507 msec 2024-11-12T18:14:22,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-12T18:14:22,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-12T18:14:22,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-12T18:14:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-12T18:14:22,808 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:22,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336b60d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb77e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,816 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,818 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d693d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f4c57e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5223b4fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de79670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b4792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,829 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5547e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,831 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf07ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,837 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15069950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,839 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cfd1c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4810677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,842 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966ad20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,844 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550d85d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,846 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e473665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:22,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:22,847 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed8cfb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:22,849 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e246ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:22,850 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-12T18:14:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T18:14:22,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,853 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:22,854 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:22,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,861 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,861 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,861 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,862 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,863 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,873 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,881 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:14:22,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:22,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:22,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:22,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:22,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:22,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:22,900 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:22,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:14:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T18:14:22,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435262949, max timestamp=1731435262969Num uniques 859; 2024-11-12T18:14:22,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126a8b88fc56d24e11908ce5f4f7ad33b3_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435262885/Put/seqid=0 2024-11-12T18:14:22,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:22,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435262949, max timestamp=1731435262969Num uniques 859; 2024-11-12T18:14:22,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435262949, max timestamp=1731435262969Num uniques 859; 2024-11-12T18:14:22,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:22,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:23,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741948_1124 (size=12154) 2024-11-12T18:14:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:23,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126a8b88fc56d24e11908ce5f4f7ad33b3_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126a8b88fc56d24e11908ce5f4f7ad33b3_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:23,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/eb1377fa04a84449b875c290e339c6ab, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:23,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/eb1377fa04a84449b875c290e339c6ab is 175, key is test_row_0/A:col10/1731435262885/Put/seqid=0 2024-11-12T18:14:23,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741949_1125 (size=30955) 2024-11-12T18:14:23,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435323050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435323052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435323054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435263049, max timestamp=1731435263054Num uniques 1018; 2024-11-12T18:14:23,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435263049, max timestamp=1731435263054Num uniques 1018; 2024-11-12T18:14:23,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T18:14:23,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T18:14:23,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435263049, max timestamp=1731435263054Num uniques 1018; 2024-11-12T18:14:23,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T18:14:23,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T18:14:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435323172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435323172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435323182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435323187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435323189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435323311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435323311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435323311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435323402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435323402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/eb1377fa04a84449b875c290e339c6ab 2024-11-12T18:14:23,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/bed694c2c5824fc7a87d3991e27322eb is 50, key is test_row_0/B:col10/1731435262885/Put/seqid=0 2024-11-12T18:14:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T18:14:23,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435323522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435323523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435323523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741950_1126 (size=12001) 2024-11-12T18:14:23,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/bed694c2c5824fc7a87d3991e27322eb 2024-11-12T18:14:23,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/618fd34f355d48db9ca1a7a6eeae1d96 is 50, key is test_row_0/C:col10/1731435262885/Put/seqid=0 2024-11-12T18:14:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741951_1127 (size=12001) 2024-11-12T18:14:23,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/618fd34f355d48db9ca1a7a6eeae1d96 2024-11-12T18:14:23,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/eb1377fa04a84449b875c290e339c6ab as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab 2024-11-12T18:14:23,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab, entries=150, sequenceid=14, filesize=30.2 K 2024-11-12T18:14:23,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/bed694c2c5824fc7a87d3991e27322eb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bed694c2c5824fc7a87d3991e27322eb 2024-11-12T18:14:23,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bed694c2c5824fc7a87d3991e27322eb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T18:14:23,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/618fd34f355d48db9ca1a7a6eeae1d96 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/618fd34f355d48db9ca1a7a6eeae1d96 2024-11-12T18:14:23,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/618fd34f355d48db9ca1a7a6eeae1d96, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T18:14:23,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ae79f5068ec1983ef2be0dd82f4139ba in 778ms, sequenceid=14, compaction requested=false 2024-11-12T18:14:23,676 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T18:14:23,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731435263168, max timestamp=1731435263714Num uniques 1027; 2024-11-12T18:14:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731435263168, max timestamp=1731435263714Num uniques 1027; 2024-11-12T18:14:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T18:14:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T18:14:23,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731435263168, max timestamp=1731435263714Num uniques 1027; 2024-11-12T18:14:23,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T18:14:23,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=46.96 KB heapSize=124.31 KB 2024-11-12T18:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:23,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:23,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111260518e1974384a5c9b94b40ab512f1a5_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435263182/Put/seqid=0 2024-11-12T18:14:23,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741952_1128 (size=12154) 2024-11-12T18:14:23,778 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:23,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,788 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111260518e1974384a5c9b94b40ab512f1a5_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260518e1974384a5c9b94b40ab512f1a5_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:23,791 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/7a5edef979c347a583cf912960486f93, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:23,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/7a5edef979c347a583cf912960486f93 is 175, key is test_row_0/A:col10/1731435263182/Put/seqid=0 2024-11-12T18:14:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741953_1129 (size=30955) 2024-11-12T18:14:23,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731435263783, max timestamp=1731435263843Num uniques 859; 2024-11-12T18:14:23,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:23,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435263783, max timestamp=1731435263843Num uniques 859; 2024-11-12T18:14:23,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:23,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435263783, max timestamp=1731435263843Num uniques 859; 2024-11-12T18:14:23,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:23,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435323890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435323895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435323899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435323901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435323902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:23,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:23,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:23,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T18:14:24,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435324011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435324011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435324011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435324011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435324021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:24,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:24,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:24,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:24,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:24,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:24,200 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=72, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/7a5edef979c347a583cf912960486f93 2024-11-12T18:14:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435324221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435324221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/3d30b1497e8b404f95cfb93a82cc0823 is 50, key is test_row_0/B:col10/1731435263182/Put/seqid=0 2024-11-12T18:14:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435324232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435324232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435324232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:24,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:24,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741954_1130 (size=12001) 2024-11-12T18:14:24,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/3d30b1497e8b404f95cfb93a82cc0823 2024-11-12T18:14:24,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/1f4f830898384807a294ecaa67a848d4 is 50, key is test_row_0/C:col10/1731435263182/Put/seqid=0 2024-11-12T18:14:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741955_1131 (size=12001) 2024-11-12T18:14:24,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/1f4f830898384807a294ecaa67a848d4 2024-11-12T18:14:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/7a5edef979c347a583cf912960486f93 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93 2024-11-12T18:14:24,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93, entries=150, sequenceid=72, filesize=30.2 K 2024-11-12T18:14:24,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/3d30b1497e8b404f95cfb93a82cc0823 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/3d30b1497e8b404f95cfb93a82cc0823 2024-11-12T18:14:24,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/3d30b1497e8b404f95cfb93a82cc0823, entries=150, sequenceid=72, filesize=11.7 K 2024-11-12T18:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/1f4f830898384807a294ecaa67a848d4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1f4f830898384807a294ecaa67a848d4 2024-11-12T18:14:24,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1f4f830898384807a294ecaa67a848d4, entries=150, sequenceid=72, filesize=11.7 K 2024-11-12T18:14:24,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~124.27 KB/127248, currentSize=140.89 KB/144270 for ae79f5068ec1983ef2be0dd82f4139ba in 643ms, sequenceid=72, compaction requested=false 2024-11-12T18:14:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:24,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T18:14:24,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:24,406 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-12T18:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:24,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129e5a2b07f8944b0b978c961fb8ae44ac_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435263890/Put/seqid=0 2024-11-12T18:14:24,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741956_1132 (size=12154) 2024-11-12T18:14:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:24,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435324541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435324541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435324544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435324551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435324556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435324661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435324661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435324670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:24,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129e5a2b07f8944b0b978c961fb8ae44ac_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e5a2b07f8944b0b978c961fb8ae44ac_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/36cb8ca0c0074578a230ac60a0d9574e, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:24,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/36cb8ca0c0074578a230ac60a0d9574e is 175, key is test_row_0/A:col10/1731435263890/Put/seqid=0 2024-11-12T18:14:24,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741957_1133 (size=30955) 2024-11-12T18:14:24,863 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/36cb8ca0c0074578a230ac60a0d9574e 2024-11-12T18:14:24,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435324871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435324871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/73548bdae37e44638c8c500cf9d86a39 is 50, key is test_row_0/B:col10/1731435263890/Put/seqid=0 2024-11-12T18:14:24,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435324881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:24,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741958_1134 (size=12001) 2024-11-12T18:14:24,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/73548bdae37e44638c8c500cf9d86a39 2024-11-12T18:14:24,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/dc3abcfacbfd438b8c540735d15386de is 50, key is test_row_0/C:col10/1731435263890/Put/seqid=0 2024-11-12T18:14:24,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741959_1135 (size=12001) 2024-11-12T18:14:24,931 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/dc3abcfacbfd438b8c540735d15386de 2024-11-12T18:14:24,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/36cb8ca0c0074578a230ac60a0d9574e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e 2024-11-12T18:14:24,950 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e, entries=150, sequenceid=108, filesize=30.2 K 2024-11-12T18:14:24,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/73548bdae37e44638c8c500cf9d86a39 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/73548bdae37e44638c8c500cf9d86a39 2024-11-12T18:14:24,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/73548bdae37e44638c8c500cf9d86a39, entries=150, sequenceid=108, filesize=11.7 K 2024-11-12T18:14:24,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/dc3abcfacbfd438b8c540735d15386de as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/dc3abcfacbfd438b8c540735d15386de 2024-11-12T18:14:24,976 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/dc3abcfacbfd438b8c540735d15386de, entries=150, sequenceid=108, filesize=11.7 K 2024-11-12T18:14:24,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for ae79f5068ec1983ef2be0dd82f4139ba in 572ms, sequenceid=108, compaction requested=true 2024-11-12T18:14:24,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:24,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:24,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-12T18:14:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-12T18:14:24,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-12T18:14:24,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1270 sec 2024-11-12T18:14:24,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.1390 sec 2024-11-12T18:14:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T18:14:25,000 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:25,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-12T18:14:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T18:14:25,003 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:25,004 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:25,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:25,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T18:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:25,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:25,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ae2f122d5794449d8ca44d21a0fec035_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435264535/Put/seqid=0 2024-11-12T18:14:25,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741960_1136 (size=12154) 2024-11-12T18:14:25,094 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:25,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ae2f122d5794449d8ca44d21a0fec035_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ae2f122d5794449d8ca44d21a0fec035_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:25,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/be53f6b188a344038b849b8a7bd368e7, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:25,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/be53f6b188a344038b849b8a7bd368e7 is 175, key is test_row_0/A:col10/1731435264535/Put/seqid=0 2024-11-12T18:14:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T18:14:25,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741961_1137 (size=30955) 2024-11-12T18:14:25,123 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/be53f6b188a344038b849b8a7bd368e7 2024-11-12T18:14:25,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c203b1a07ff64e40a36ca80d0559d219 is 50, key is test_row_0/B:col10/1731435264535/Put/seqid=0 2024-11-12T18:14:25,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435325138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435325142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741962_1138 (size=12001) 2024-11-12T18:14:25,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T18:14:25,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435325182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435325182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435325201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435325250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435325250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T18:14:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:25,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T18:14:25,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435325460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435325460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T18:14:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:25,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c203b1a07ff64e40a36ca80d0559d219 2024-11-12T18:14:25,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/6637477017bf4760b37d48c20de42f9d is 50, key is test_row_0/C:col10/1731435264535/Put/seqid=0 2024-11-12T18:14:25,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741963_1139 (size=12001) 2024-11-12T18:14:25,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/6637477017bf4760b37d48c20de42f9d 2024-11-12T18:14:25,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/be53f6b188a344038b849b8a7bd368e7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7 2024-11-12T18:14:25,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7, entries=150, sequenceid=120, filesize=30.2 K 2024-11-12T18:14:25,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c203b1a07ff64e40a36ca80d0559d219 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c203b1a07ff64e40a36ca80d0559d219 2024-11-12T18:14:25,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c203b1a07ff64e40a36ca80d0559d219, entries=150, sequenceid=120, filesize=11.7 K 2024-11-12T18:14:25,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/6637477017bf4760b37d48c20de42f9d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6637477017bf4760b37d48c20de42f9d 2024-11-12T18:14:25,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6637477017bf4760b37d48c20de42f9d, entries=150, sequenceid=120, filesize=11.7 K 2024-11-12T18:14:25,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae79f5068ec1983ef2be0dd82f4139ba in 551ms, sequenceid=120, compaction requested=true 2024-11-12T18:14:25,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:25,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:25,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:25,615 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:25,615 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:25,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:25,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:25,618 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:25,618 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:25,618 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,618 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bed694c2c5824fc7a87d3991e27322eb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/3d30b1497e8b404f95cfb93a82cc0823, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/73548bdae37e44638c8c500cf9d86a39, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c203b1a07ff64e40a36ca80d0559d219] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=46.9 K 2024-11-12T18:14:25,618 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:25,619 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:25,619 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,619 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=120.9 K 2024-11-12T18:14:25,619 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,619 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7] 2024-11-12T18:14:25,621 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting bed694c2c5824fc7a87d3991e27322eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731435262885 2024-11-12T18:14:25,621 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1377fa04a84449b875c290e339c6ab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731435262885 2024-11-12T18:14:25,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,622 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:25,627 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d30b1497e8b404f95cfb93a82cc0823, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731435263182 2024-11-12T18:14:25,627 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a5edef979c347a583cf912960486f93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731435263182 2024-11-12T18:14:25,628 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 73548bdae37e44638c8c500cf9d86a39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731435263870 2024-11-12T18:14:25,628 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36cb8ca0c0074578a230ac60a0d9574e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731435263870 2024-11-12T18:14:25,629 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c203b1a07ff64e40a36ca80d0559d219, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731435264535 2024-11-12T18:14:25,629 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting be53f6b188a344038b849b8a7bd368e7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731435264535 2024-11-12T18:14:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T18:14:25,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111265e85d6d4aab42b0947aac3e9b537d8d_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435265123/Put/seqid=0 2024-11-12T18:14:25,664 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:25,667 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#121 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:25,668 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/65cc9fb3c6574298a1c48ff2e55ec9a1 is 50, key is test_row_0/B:col10/1731435264535/Put/seqid=0 2024-11-12T18:14:25,671 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e585c220a2ef4596864857656dd263fd_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:25,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741964_1140 (size=12304) 2024-11-12T18:14:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741965_1141 (size=12139) 2024-11-12T18:14:25,687 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111265e85d6d4aab42b0947aac3e9b537d8d_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111265e85d6d4aab42b0947aac3e9b537d8d_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:25,687 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e585c220a2ef4596864857656dd263fd_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:25,687 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e585c220a2ef4596864857656dd263fd_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:25,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9343cfa5febd41e4bcd94357a54e466f, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:25,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9343cfa5febd41e4bcd94357a54e466f is 175, key is test_row_0/A:col10/1731435265123/Put/seqid=0 2024-11-12T18:14:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:25,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741967_1143 (size=31105) 2024-11-12T18:14:25,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9343cfa5febd41e4bcd94357a54e466f 2024-11-12T18:14:25,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435325733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/f1ba52fac51d4b86bad7c1d2ee801379 is 50, key is test_row_0/B:col10/1731435265123/Put/seqid=0 2024-11-12T18:14:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741966_1142 (size=4469) 2024-11-12T18:14:25,745 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:14:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435325736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435325738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435325770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435325770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741968_1144 (size=12151) 2024-11-12T18:14:25,790 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/f1ba52fac51d4b86bad7c1d2ee801379 2024-11-12T18:14:25,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c38aa06546e34735a12a7054cc1f1012 is 50, key is test_row_0/C:col10/1731435265123/Put/seqid=0 2024-11-12T18:14:25,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741969_1145 (size=12151) 2024-11-12T18:14:25,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c38aa06546e34735a12a7054cc1f1012 2024-11-12T18:14:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9343cfa5febd41e4bcd94357a54e466f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f 2024-11-12T18:14:25,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435325840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,842 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f, entries=150, sequenceid=140, filesize=30.4 K 2024-11-12T18:14:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/f1ba52fac51d4b86bad7c1d2ee801379 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f1ba52fac51d4b86bad7c1d2ee801379 2024-11-12T18:14:25,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f1ba52fac51d4b86bad7c1d2ee801379, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T18:14:25,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c38aa06546e34735a12a7054cc1f1012 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c38aa06546e34735a12a7054cc1f1012 2024-11-12T18:14:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435325851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435325851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:25,860 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c38aa06546e34735a12a7054cc1f1012, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T18:14:25,861 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae79f5068ec1983ef2be0dd82f4139ba in 239ms, sequenceid=140, compaction requested=true 2024-11-12T18:14:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-12T18:14:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-12T18:14:25,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-12T18:14:25,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 860 msec 2024-11-12T18:14:25,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 866 msec 2024-11-12T18:14:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:26,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:26,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:26,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f43079cb8f234b6ab92858f3c357ca25_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435265734/Put/seqid=0 2024-11-12T18:14:26,084 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/65cc9fb3c6574298a1c48ff2e55ec9a1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/65cc9fb3c6574298a1c48ff2e55ec9a1 2024-11-12T18:14:26,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741970_1146 (size=14794) 2024-11-12T18:14:26,097 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into 65cc9fb3c6574298a1c48ff2e55ec9a1(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:26,097 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:26,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:26,097 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=12, startTime=1731435265615; duration=0sec 2024-11-12T18:14:26,097 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:26,098 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:26,098 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T18:14:26,102 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60155 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T18:14:26,102 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:26,102 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,102 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/618fd34f355d48db9ca1a7a6eeae1d96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1f4f830898384807a294ecaa67a848d4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/dc3abcfacbfd438b8c540735d15386de, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6637477017bf4760b37d48c20de42f9d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c38aa06546e34735a12a7054cc1f1012] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=58.7 K 2024-11-12T18:14:26,103 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 618fd34f355d48db9ca1a7a6eeae1d96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731435262885 2024-11-12T18:14:26,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f43079cb8f234b6ab92858f3c357ca25_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f43079cb8f234b6ab92858f3c357ca25_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:26,104 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4f830898384807a294ecaa67a848d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1731435263182 2024-11-12T18:14:26,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/fbdfc8d748db43daae3a699463ad272a, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:26,105 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting dc3abcfacbfd438b8c540735d15386de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731435263870 2024-11-12T18:14:26,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/fbdfc8d748db43daae3a699463ad272a is 175, key is test_row_0/A:col10/1731435265734/Put/seqid=0 2024-11-12T18:14:26,106 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6637477017bf4760b37d48c20de42f9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731435264535 2024-11-12T18:14:26,108 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c38aa06546e34735a12a7054cc1f1012, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731435265123 2024-11-12T18:14:26,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435326119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435326123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435326123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,134 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#126 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:26,135 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8ad0456cd20740a6919945e6dfb29f5f is 50, key is test_row_0/C:col10/1731435265123/Put/seqid=0 2024-11-12T18:14:26,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741971_1147 (size=39749) 2024-11-12T18:14:26,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/fbdfc8d748db43daae3a699463ad272a 2024-11-12T18:14:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T18:14:26,141 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:26,142 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T18:14:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T18:14:26,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:26,145 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:26,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:26,153 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#122 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:26,154 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/fcf2de00af6f4748a1b498078fe30658 is 175, key is test_row_0/A:col10/1731435264535/Put/seqid=0 2024-11-12T18:14:26,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741972_1148 (size=12323) 2024-11-12T18:14:26,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/b7b0ebf7f5574351962262e189fdf20f is 50, key is test_row_0/B:col10/1731435265734/Put/seqid=0 2024-11-12T18:14:26,174 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8ad0456cd20740a6919945e6dfb29f5f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8ad0456cd20740a6919945e6dfb29f5f 2024-11-12T18:14:26,188 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into 8ad0456cd20740a6919945e6dfb29f5f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:26,188 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:26,188 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=11, startTime=1731435265616; duration=0sec 2024-11-12T18:14:26,188 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:26,188 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741974_1150 (size=12151) 2024-11-12T18:14:26,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/b7b0ebf7f5574351962262e189fdf20f 2024-11-12T18:14:26,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741973_1149 (size=31093) 2024-11-12T18:14:26,222 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/fcf2de00af6f4748a1b498078fe30658 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fcf2de00af6f4748a1b498078fe30658 2024-11-12T18:14:26,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c80e0e707e9d4ba0b0a3b5a7d1678494 is 50, key is test_row_0/C:col10/1731435265734/Put/seqid=0 2024-11-12T18:14:26,234 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into fcf2de00af6f4748a1b498078fe30658(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:26,234 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:26,234 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=12, startTime=1731435265615; duration=0sec 2024-11-12T18:14:26,234 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:26,234 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:26,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435326241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435326241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435326241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T18:14:26,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741975_1151 (size=12151) 2024-11-12T18:14:26,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c80e0e707e9d4ba0b0a3b5a7d1678494 2024-11-12T18:14:26,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/fbdfc8d748db43daae3a699463ad272a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a 2024-11-12T18:14:26,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a, entries=200, sequenceid=153, filesize=38.8 K 2024-11-12T18:14:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/b7b0ebf7f5574351962262e189fdf20f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/b7b0ebf7f5574351962262e189fdf20f 2024-11-12T18:14:26,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/b7b0ebf7f5574351962262e189fdf20f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T18:14:26,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c80e0e707e9d4ba0b0a3b5a7d1678494 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c80e0e707e9d4ba0b0a3b5a7d1678494 2024-11-12T18:14:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435326291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435326291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c80e0e707e9d4ba0b0a3b5a7d1678494, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T18:14:26,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T18:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:26,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:26,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ae79f5068ec1983ef2be0dd82f4139ba in 246ms, sequenceid=153, compaction requested=true 2024-11-12T18:14:26,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:26,301 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:26,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:26,303 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:26,303 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:26,303 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,303 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fcf2de00af6f4748a1b498078fe30658, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=99.6 K 2024-11-12T18:14:26,303 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,303 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fcf2de00af6f4748a1b498078fe30658, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a] 2024-11-12T18:14:26,304 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf2de00af6f4748a1b498078fe30658, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731435264535 2024-11-12T18:14:26,304 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9343cfa5febd41e4bcd94357a54e466f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731435265123 2024-11-12T18:14:26,305 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbdfc8d748db43daae3a699463ad272a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435265725 2024-11-12T18:14:26,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:26,305 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:26,307 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:26,307 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:26,307 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,307 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/65cc9fb3c6574298a1c48ff2e55ec9a1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f1ba52fac51d4b86bad7c1d2ee801379, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/b7b0ebf7f5574351962262e189fdf20f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=35.6 K 2024-11-12T18:14:26,308 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 65cc9fb3c6574298a1c48ff2e55ec9a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731435264535 2024-11-12T18:14:26,309 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ba52fac51d4b86bad7c1d2ee801379, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731435265123 2024-11-12T18:14:26,310 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b0ebf7f5574351962262e189fdf20f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435265725 2024-11-12T18:14:26,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:26,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:26,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:26,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:26,321 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:26,329 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#130 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:26,331 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/bbbfe68ae9bd4829a56df750decca2c4 is 50, key is test_row_0/B:col10/1731435265734/Put/seqid=0 2024-11-12T18:14:26,332 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111206f5d24fa2e847c6874b4ccb036d6ef7_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:26,336 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111206f5d24fa2e847c6874b4ccb036d6ef7_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:26,336 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111206f5d24fa2e847c6874b4ccb036d6ef7_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:26,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741976_1152 (size=12391) 2024-11-12T18:14:26,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741977_1153 (size=4469) 2024-11-12T18:14:26,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T18:14:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:26,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:14:26,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435266117, max timestamp=1731435266452Num uniques 859; 2024-11-12T18:14:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435266117, max timestamp=1731435266452Num uniques 859; 2024-11-12T18:14:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435266117, max timestamp=1731435266452Num uniques 859; 2024-11-12T18:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T18:14:26,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127978dd52ac11478e854fcd29a5a3eb31_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435266119/Put/seqid=0 2024-11-12T18:14:26,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741978_1154 (size=12304) 2024-11-12T18:14:26,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127978dd52ac11478e854fcd29a5a3eb31_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127978dd52ac11478e854fcd29a5a3eb31_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:26,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/d0e147150c7b42c984be3faffb8e7d91, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435266483, max timestamp=1731435266507Num uniques 859; 2024-11-12T18:14:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435266483, max timestamp=1731435266507Num uniques 859; 2024-11-12T18:14:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435266483, max timestamp=1731435266507Num uniques 859; 2024-11-12T18:14:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/d0e147150c7b42c984be3faffb8e7d91 is 175, key is test_row_0/A:col10/1731435266119/Put/seqid=0 2024-11-12T18:14:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:26,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741979_1155 (size=31105) 2024-11-12T18:14:26,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435266558, max timestamp=1731435266566Num uniques 1068; 2024-11-12T18:14:26,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435266558, max timestamp=1731435266566Num uniques 1068; 2024-11-12T18:14:26,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T18:14:26,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T18:14:26,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435266558, max timestamp=1731435266566Num uniques 1068; 2024-11-12T18:14:26,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T18:14:26,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731435266609, max timestamp=1731435266624Num uniques 1027; 2024-11-12T18:14:26,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T18:14:26,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731435266609, max timestamp=1731435266624Num uniques 1027; 2024-11-12T18:14:26,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T18:14:26,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731435266609, max timestamp=1731435266624Num uniques 1027; 2024-11-12T18:14:26,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T18:14:26,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731435266640, max timestamp=1731435266668Num uniques 1086; 2024-11-12T18:14:26,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731435266640, max timestamp=1731435266668Num uniques 1086; 2024-11-12T18:14:26,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731435266640, max timestamp=1731435266668Num uniques 1086; 2024-11-12T18:14:26,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T18:14:26,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T18:14:26,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T18:14:26,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731435266687, max timestamp=1731435266718Num uniques 1095; 2024-11-12T18:14:26,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-12T18:14:26,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731435266687, max timestamp=1731435266718Num uniques 1095; 2024-11-12T18:14:26,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-12T18:14:26,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731435266687, max timestamp=1731435266718Num uniques 1095; 2024-11-12T18:14:26,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-12T18:14:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T18:14:26,776 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/bbbfe68ae9bd4829a56df750decca2c4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bbbfe68ae9bd4829a56df750decca2c4 2024-11-12T18:14:26,777 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#129 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:26,778 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/e4ff9a8b7e994851ab58ea3258fb04dd is 175, key is test_row_0/A:col10/1731435265734/Put/seqid=0 2024-11-12T18:14:26,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435326779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,787 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into bbbfe68ae9bd4829a56df750decca2c4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:26,787 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:26,787 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435266305; duration=0sec 2024-11-12T18:14:26,787 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:26,787 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:26,787 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:14:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,788 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:26,788 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:26,788 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. because compaction request was cancelled 2024-11-12T18:14:26,788 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:26,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741980_1156 (size=31345) 2024-11-12T18:14:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435326782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435326782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,806 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/e4ff9a8b7e994851ab58ea3258fb04dd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e4ff9a8b7e994851ab58ea3258fb04dd 2024-11-12T18:14:26,814 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into e4ff9a8b7e994851ab58ea3258fb04dd(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:26,814 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:26,814 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435266301; duration=0sec 2024-11-12T18:14:26,814 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:26,814 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:26,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435326891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435326900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435326900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:26,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/d0e147150c7b42c984be3faffb8e7d91 2024-11-12T18:14:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c63c3715dc7849b3962881138a854ea7 is 50, key is test_row_0/B:col10/1731435266119/Put/seqid=0 2024-11-12T18:14:26,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741981_1157 (size=12151) 2024-11-12T18:14:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435327100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435327110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435327110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T18:14:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435327300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,311 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435327314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,316 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,351 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c63c3715dc7849b3962881138a854ea7 2024-11-12T18:14:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/d4f0dc9e95ef43dfa28c6b332d14d5b1 is 50, key is test_row_0/C:col10/1731435266119/Put/seqid=0 2024-11-12T18:14:27,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741982_1158 (size=12151) 2024-11-12T18:14:27,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435327411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435327420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435327430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/d4f0dc9e95ef43dfa28c6b332d14d5b1 2024-11-12T18:14:27,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/d0e147150c7b42c984be3faffb8e7d91 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91 2024-11-12T18:14:27,793 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91, entries=150, sequenceid=177, filesize=30.4 K 2024-11-12T18:14:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c63c3715dc7849b3962881138a854ea7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c63c3715dc7849b3962881138a854ea7 2024-11-12T18:14:27,803 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c63c3715dc7849b3962881138a854ea7, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T18:14:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/d4f0dc9e95ef43dfa28c6b332d14d5b1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d4f0dc9e95ef43dfa28c6b332d14d5b1 2024-11-12T18:14:27,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d4f0dc9e95ef43dfa28c6b332d14d5b1, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T18:14:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-12T18:14:27,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=167.72 KB/171750 for ae79f5068ec1983ef2be0dd82f4139ba in 1357ms, sequenceid=177, compaction requested=true 2024-11-12T18:14:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T18:14:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T18:14:27,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T18:14:27,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6670 sec 2024-11-12T18:14:27,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6720 sec 2024-11-12T18:14:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:27,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=174.43 KB heapSize=483.09 KB 2024-11-12T18:14:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:14:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:14:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:14:27,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112df688f168c1048c68bb692282c0dff51_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:27,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435327937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741983_1159 (size=14994) 2024-11-12T18:14:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435327938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:27,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:27,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112df688f168c1048c68bb692282c0dff51_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112df688f168c1048c68bb692282c0dff51_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:27,946 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/85d30712cbed42fc855b6eaf7c9ea593, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:27,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/85d30712cbed42fc855b6eaf7c9ea593 is 175, key is test_row_0/A:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741984_1160 (size=39949) 2024-11-12T18:14:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435327950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435328041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435328050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435328251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435328270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T18:14:28,291 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:28,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-12T18:14:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T18:14:28,296 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:28,297 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:28,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:28,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/85d30712cbed42fc855b6eaf7c9ea593 2024-11-12T18:14:28,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/7b2a56884a684c1a93d6cd3f0f54787f is 50, key is test_row_0/B:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:28,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741985_1161 (size=12301) 2024-11-12T18:14:28,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/7b2a56884a684c1a93d6cd3f0f54787f 2024-11-12T18:14:28,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c481cb34c8de421cb4b3c831092c51b0 is 50, key is test_row_0/C:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T18:14:28,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741986_1162 (size=12301) 2024-11-12T18:14:28,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T18:14:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435328560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435328580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:28,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:28,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T18:14:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T18:14:28,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T18:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:28,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c481cb34c8de421cb4b3c831092c51b0 2024-11-12T18:14:28,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/85d30712cbed42fc855b6eaf7c9ea593 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593 2024-11-12T18:14:28,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593, entries=200, sequenceid=286, filesize=39.0 K 2024-11-12T18:14:28,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/7b2a56884a684c1a93d6cd3f0f54787f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/7b2a56884a684c1a93d6cd3f0f54787f 2024-11-12T18:14:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/7b2a56884a684c1a93d6cd3f0f54787f, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T18:14:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c481cb34c8de421cb4b3c831092c51b0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c481cb34c8de421cb4b3c831092c51b0 2024-11-12T18:14:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c481cb34c8de421cb4b3c831092c51b0, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T18:14:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~483.05 KB/494640, currentSize=13.42 KB/13740 for ae79f5068ec1983ef2be0dd82f4139ba in 915ms, sequenceid=286, compaction requested=true 2024-11-12T18:14:28,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:28,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:28,837 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:28,838 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:28,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:28,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:28,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:28,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,839 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:28,839 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,839 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:28,840 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,840 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bbbfe68ae9bd4829a56df750decca2c4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c63c3715dc7849b3962881138a854ea7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/7b2a56884a684c1a93d6cd3f0f54787f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.0 K 2024-11-12T18:14:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,840 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:28,840 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,840 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e4ff9a8b7e994851ab58ea3258fb04dd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=100.0 K 2024-11-12T18:14:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,840 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,840 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e4ff9a8b7e994851ab58ea3258fb04dd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593] 2024-11-12T18:14:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,840 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting bbbfe68ae9bd4829a56df750decca2c4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435265725 2024-11-12T18:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,841 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ff9a8b7e994851ab58ea3258fb04dd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435265725 2024-11-12T18:14:28,841 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c63c3715dc7849b3962881138a854ea7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731435266117 2024-11-12T18:14:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,842 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e147150c7b42c984be3faffb8e7d91, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731435266117 2024-11-12T18:14:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,842 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2a56884a684c1a93d6cd3f0f54787f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731435266753 2024-11-12T18:14:28,842 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d30712cbed42fc855b6eaf7c9ea593, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731435266753 2024-11-12T18:14:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,855 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,855 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/fab9cb788c044110bf93b45c7756807d is 50, key is test_row_0/B:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,857 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,861 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111212204929353b4642971f42cceddeee7c_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,864 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111212204929353b4642971f42cceddeee7c_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,864 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111212204929353b4642971f42cceddeee7c_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741987_1163 (size=12643) 2024-11-12T18:14:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741988_1164 (size=4469) 2024-11-12T18:14:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,889 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#138 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,890 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/e730d1c86c024399bd327d8e7fa7786d is 175, key is test_row_0/A:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741989_1165 (size=31597) 2024-11-12T18:14:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,909 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/e730d1c86c024399bd327d8e7fa7786d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e730d1c86c024399bd327d8e7fa7786d 2024-11-12T18:14:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T18:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,915 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-12T18:14:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,918 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into e730d1c86c024399bd327d8e7fa7786d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,918 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:28,918 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435268837; duration=0sec 2024-11-12T18:14:28,918 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,918 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:28,918 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:28,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T18:14:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,922 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48926 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:28,922 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:28,922 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:28,922 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8ad0456cd20740a6919945e6dfb29f5f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c80e0e707e9d4ba0b0a3b5a7d1678494, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d4f0dc9e95ef43dfa28c6b332d14d5b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c481cb34c8de421cb4b3c831092c51b0] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=47.8 K 2024-11-12T18:14:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,923 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad0456cd20740a6919945e6dfb29f5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731435265123 2024-11-12T18:14:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,923 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80e0e707e9d4ba0b0a3b5a7d1678494, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435265725 2024-11-12T18:14:28,924 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f0dc9e95ef43dfa28c6b332d14d5b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731435266117 2024-11-12T18:14:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,924 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c481cb34c8de421cb4b3c831092c51b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731435266753 2024-11-12T18:14:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112069640ae20c245f18992eddc49f8fbea_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435267935/Put/seqid=0 2024-11-12T18:14:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,947 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:28,947 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/be3d5189874a43aaafe4e0c779e44cc1 is 50, key is test_row_0/C:col10/1731435267920/Put/seqid=0 2024-11-12T18:14:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741990_1166 (size=9914) 2024-11-12T18:14:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741991_1167 (size=12609) 2024-11-12T18:14:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,007 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/be3d5189874a43aaafe4e0c779e44cc1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/be3d5189874a43aaafe4e0c779e44cc1 2024-11-12T18:14:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,014 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into be3d5189874a43aaafe4e0c779e44cc1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:29,014 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:29,014 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=12, startTime=1731435268838; duration=0sec 2024-11-12T18:14:29,014 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:29,014 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:29,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731435269095, max timestamp=1731435269119Num uniques 859; 2024-11-12T18:14:29,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-12T18:14:29,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731435269095, max timestamp=1731435269119Num uniques 859; 2024-11-12T18:14:29,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-12T18:14:29,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731435269095, max timestamp=1731435269119Num uniques 859; 2024-11-12T18:14:29,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-12T18:14:29,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731435269150, max timestamp=1731435269196Num uniques 1268; 2024-11-12T18:14:29,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731435269150, max timestamp=1731435269196Num uniques 1268; 2024-11-12T18:14:29,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=14558, suffix heap size=159840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=318 2024-11-12T18:14:29,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=14558, suffix heap size=159840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=318 2024-11-12T18:14:29,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731435269150, max timestamp=1731435269196Num uniques 1268; 2024-11-12T18:14:29,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=14558, suffix heap size=159840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=318 2024-11-12T18:14:29,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731435269222, max timestamp=1731435269239Num uniques 1177; 2024-11-12T18:14:29,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-12T18:14:29,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731435269222, max timestamp=1731435269239Num uniques 1177; 2024-11-12T18:14:29,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-12T18:14:29,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731435269222, max timestamp=1731435269239Num uniques 1177; 2024-11-12T18:14:29,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-12T18:14:29,279 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/fab9cb788c044110bf93b45c7756807d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/fab9cb788c044110bf93b45c7756807d 2024-11-12T18:14:29,288 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into fab9cb788c044110bf93b45c7756807d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:29,288 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:29,288 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435268838; duration=0sec 2024-11-12T18:14:29,288 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:29,288 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:29,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731435269264, max timestamp=1731435269287Num uniques 1136; 2024-11-12T18:14:29,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731435269264, max timestamp=1731435269287Num uniques 1136; 2024-11-12T18:14:29,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-12T18:14:29,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-12T18:14:29,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731435269264, max timestamp=1731435269287Num uniques 1136; 2024-11-12T18:14:29,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-12T18:14:29,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731435269317, max timestamp=1731435269332Num uniques 1195; 2024-11-12T18:14:29,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-12T18:14:29,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731435269317, max timestamp=1731435269332Num uniques 1195; 2024-11-12T18:14:29,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731435269317, max timestamp=1731435269332Num uniques 1195; 2024-11-12T18:14:29,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-12T18:14:29,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-12T18:14:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:29,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731435269353, max timestamp=1731435269371Num uniques 1154; 2024-11-12T18:14:29,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T18:14:29,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731435269353, max timestamp=1731435269371Num uniques 1154; 2024-11-12T18:14:29,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T18:14:29,398 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112069640ae20c245f18992eddc49f8fbea_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112069640ae20c245f18992eddc49f8fbea_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:29,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731435269353, max timestamp=1731435269371Num uniques 1154; 2024-11-12T18:14:29,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-12T18:14:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/ced8c53db9284678b12cca2d44d4ec31, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/ced8c53db9284678b12cca2d44d4ec31 is 175, key is test_row_0/A:col10/1731435267935/Put/seqid=0 2024-11-12T18:14:29,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741992_1168 (size=22561) 2024-11-12T18:14:29,421 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/ced8c53db9284678b12cca2d44d4ec31 2024-11-12T18:14:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T18:14:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/aae86d7a11d64ef693c31354f09ccee4 is 50, key is test_row_0/B:col10/1731435267935/Put/seqid=0 2024-11-12T18:14:29,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731435269420, max timestamp=1731435269434Num uniques 1013; 2024-11-12T18:14:29,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T18:14:29,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731435269420, max timestamp=1731435269434Num uniques 1013; 2024-11-12T18:14:29,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T18:14:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741993_1169 (size=9857) 2024-11-12T18:14:29,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731435269420, max timestamp=1731435269434Num uniques 1013; 2024-11-12T18:14:29,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T18:14:29,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1731435269444, max timestamp=1731435269463Num uniques 1072; 2024-11-12T18:14:29,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1731435269444, max timestamp=1731435269463Num uniques 1072; 2024-11-12T18:14:29,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=17032, suffix heap size=139456,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=372 2024-11-12T18:14:29,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=17032, suffix heap size=139456,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=372 2024-11-12T18:14:29,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1731435269444, max timestamp=1731435269463Num uniques 1072; 2024-11-12T18:14:29,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=17032, suffix heap size=139456,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=372 2024-11-12T18:14:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435329496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435329497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:29,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435329498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:29,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1731435269481, max timestamp=1731435269497Num uniques 1231; 2024-11-12T18:14:29,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56376, new segment data size=15156, suffix heap size=155992,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=331 2024-11-12T18:14:29,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731435269481, max timestamp=1731435269497Num uniques 1231; 2024-11-12T18:14:29,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56366, new segment data size=8276, suffix heap size=155992,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=181 2024-11-12T18:14:29,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731435269481, max timestamp=1731435269497Num uniques 1231; 2024-11-12T18:14:29,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=56376, new segment data size=8286, suffix heap size=155992,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=181 2024-11-12T18:14:29,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731435269522, max timestamp=1731435269548Num uniques 1040; 2024-11-12T18:14:29,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731435269522, max timestamp=1731435269548Num uniques 1190; 2024-11-12T18:14:29,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T18:14:29,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=10990, suffix heap size=151728,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=240 2024-11-12T18:14:29,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731435269522, max timestamp=1731435269548Num uniques 1040; 2024-11-12T18:14:29,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T18:14:29,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731435269567, max timestamp=1731435269610Num uniques 1099; 2024-11-12T18:14:29,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-12T18:14:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731435269567, max timestamp=1731435269610Num uniques 1099; 2024-11-12T18:14:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731435269567, max timestamp=1731435269610Num uniques 1099; 2024-11-12T18:14:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-12T18:14:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-12T18:14:29,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16388, totalHeapSize=37472, min timestamp=1731435269630, max timestamp=1731435269654Num uniques 1108; 2024-11-12T18:14:29,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=16388, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-11-12T18:14:29,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435269630, max timestamp=1731435269654Num uniques 1108; 2024-11-12T18:14:29,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=6870, suffix heap size=143200,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=150 2024-11-12T18:14:29,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16388, totalHeapSize=37472, min timestamp=1731435269630, max timestamp=1731435269654Num uniques 1108; 2024-11-12T18:14:29,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=16388, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-11-12T18:14:29,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=417, cellSize=19092, totalHeapSize=43608, min timestamp=1731435269674, max timestamp=1731435269693Num uniques 1217; 2024-11-12T18:14:29,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=19092, suffix heap size=154536,new segment heap size=43608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=417 2024-11-12T18:14:29,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=417, cellSize=19092, totalHeapSize=43608, min timestamp=1731435269674, max timestamp=1731435269693Num uniques 1217; 2024-11-12T18:14:29,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=417, cellSize=19092, totalHeapSize=43608, min timestamp=1731435269674, max timestamp=1731435269693Num uniques 1009; 2024-11-12T18:14:29,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55732, new segment data size=19092, suffix heap size=154536,new segment heap size=43608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1217, new segment cells count=417 2024-11-12T18:14:29,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=19092, suffix heap size=132904,new segment heap size=43608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=417 2024-11-12T18:14:29,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435329721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:29,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435329724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:29,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731435269715, max timestamp=1731435269725Num uniques 1276; 2024-11-12T18:14:29,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=58436, new segment data size=14926, suffix heap size=160672,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1276, new segment cells count=326 2024-11-12T18:14:29,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731435269715, max timestamp=1731435269725Num uniques 1276; 2024-11-12T18:14:29,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=58436, new segment data size=14926, suffix heap size=160672,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1276, new segment cells count=326 2024-11-12T18:14:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:29,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731435269715, max timestamp=1731435269725Num uniques 1276; 2024-11-12T18:14:29,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=58436, new segment data size=14926, suffix heap size=160672,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1276, new segment cells count=326 2024-11-12T18:14:29,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731435269754, max timestamp=1731435269775Num uniques 1185; 2024-11-12T18:14:29,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731435269754, max timestamp=1731435269775Num uniques 1185; 2024-11-12T18:14:29,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=10760, suffix heap size=151208,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=235 2024-11-12T18:14:29,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=10760, suffix heap size=151208,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=235 2024-11-12T18:14:29,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731435269754, max timestamp=1731435269775Num uniques 1185; 2024-11-12T18:14:29,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=10760, suffix heap size=151208,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=235 2024-11-12T18:14:29,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731435269808, max timestamp=1731435269825Num uniques 1094; 2024-11-12T18:14:29,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731435269808, max timestamp=1731435269825Num uniques 1094; 2024-11-12T18:14:29,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T18:14:29,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T18:14:29,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731435269808, max timestamp=1731435269825Num uniques 1094; 2024-11-12T18:14:29,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T18:14:29,842 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/aae86d7a11d64ef693c31354f09ccee4 2024-11-12T18:14:29,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/6b6f846dfe074a7aa79cb3b90da539e2 is 50, key is test_row_0/C:col10/1731435267935/Put/seqid=0 2024-11-12T18:14:29,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741994_1170 (size=9857) 2024-11-12T18:14:29,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=353, cellSize=16165, totalHeapSize=36952, min timestamp=1731435269851, max timestamp=1731435269876Num uniques 1103; 2024-11-12T18:14:29,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=353, cellSize=16165, totalHeapSize=36952, min timestamp=1731435269851, max timestamp=1731435269876Num uniques 1103; 2024-11-12T18:14:29,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=16165, suffix heap size=142680,new segment heap size=36952 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=353 2024-11-12T18:14:29,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=16165, suffix heap size=142680,new segment heap size=36952 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=353 2024-11-12T18:14:29,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=353, cellSize=16165, totalHeapSize=36952, min timestamp=1731435269851, max timestamp=1731435269876Num uniques 1103; 2024-11-12T18:14:29,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=16165, suffix heap size=142680,new segment heap size=36952 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=353 2024-11-12T18:14:29,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1731435269898, max timestamp=1731435269907Num uniques 1212; 2024-11-12T18:14:29,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1731435269898, max timestamp=1731435269907Num uniques 1212; 2024-11-12T18:14:29,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55502, new segment data size=11992, suffix heap size=154016,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1212, new segment cells count=262 2024-11-12T18:14:29,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=55502, new segment data size=11992, suffix heap size=154016,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1212, new segment cells count=262 2024-11-12T18:14:29,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1731435269898, max timestamp=1731435269907Num uniques 1212; 2024-11-12T18:14:29,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55502, new segment data size=11992, suffix heap size=154016,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1212, new segment cells count=262 2024-11-12T18:14:29,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7817, totalHeapSize=18024, min timestamp=1731435269931, max timestamp=1731435269942Num uniques 1121; 2024-11-12T18:14:29,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51327, new segment data size=7817, suffix heap size=144552,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=171 2024-11-12T18:14:29,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7821, totalHeapSize=18024, min timestamp=1731435269931, max timestamp=1731435269942Num uniques 1121; 2024-11-12T18:14:29,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51331, new segment data size=7821, suffix heap size=144552,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=171 2024-11-12T18:14:29,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1731435269931, max timestamp=1731435269942Num uniques 1121; 2024-11-12T18:14:29,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=7826, suffix heap size=144552,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=171 2024-11-12T18:14:29,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8230, totalHeapSize=18960, min timestamp=1731435269970, max timestamp=1731435269979Num uniques 1030; 2024-11-12T18:14:29,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47160, new segment data size=8230, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-11-12T18:14:29,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8230, totalHeapSize=18960, min timestamp=1731435269970, max timestamp=1731435269979Num uniques 1030; 2024-11-12T18:14:29,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8230, totalHeapSize=18960, min timestamp=1731435269970, max timestamp=1731435269979Num uniques 1030; 2024-11-12T18:14:29,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47160, new segment data size=8230, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-11-12T18:14:29,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47160, new segment data size=8230, suffix heap size=135088,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=180 2024-11-12T18:14:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=439, cellSize=20104, totalHeapSize=45896, min timestamp=1731435269992, max timestamp=1731435270015Num uniques 1039; 2024-11-12T18:14:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=20104, suffix heap size=136024,new segment heap size=45896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=439 2024-11-12T18:14:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=439, cellSize=20104, totalHeapSize=45896, min timestamp=1731435269992, max timestamp=1731435270015Num uniques 1039; 2024-11-12T18:14:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=20104, suffix heap size=136024,new segment heap size=45896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=439 2024-11-12T18:14:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=439, cellSize=20104, totalHeapSize=45896, min timestamp=1731435269992, max timestamp=1731435270015Num uniques 1039; 2024-11-12T18:14:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=20104, suffix heap size=136024,new segment heap size=45896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=439 2024-11-12T18:14:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435330046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435330046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435330046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13638, totalHeapSize=31232, min timestamp=1731435270032, max timestamp=1731435270051Num uniques 1298; 2024-11-12T18:14:30,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=59438, new segment data size=13638, suffix heap size=162960,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1298, new segment cells count=298 2024-11-12T18:14:30,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13638, totalHeapSize=31232, min timestamp=1731435270032, max timestamp=1731435270051Num uniques 1298; 2024-11-12T18:14:30,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=59438, new segment data size=13638, suffix heap size=162960,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1298, new segment cells count=298 2024-11-12T18:14:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13639, totalHeapSize=31232, min timestamp=1731435270032, max timestamp=1731435270051Num uniques 1298; 2024-11-12T18:14:30,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=59439, new segment data size=13639, suffix heap size=162960,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1298, new segment cells count=298 2024-11-12T18:14:30,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=307, cellSize=14055, totalHeapSize=32168, min timestamp=1731435270082, max timestamp=1731435270109Num uniques 1157; 2024-11-12T18:14:30,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=14055, suffix heap size=148296,new segment heap size=32168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=307 2024-11-12T18:14:30,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=307, cellSize=14055, totalHeapSize=32168, min timestamp=1731435270082, max timestamp=1731435270109Num uniques 1157; 2024-11-12T18:14:30,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=14055, suffix heap size=148296,new segment heap size=32168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=307 2024-11-12T18:14:30,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1731435270082, max timestamp=1731435270109Num uniques 1157; 2024-11-12T18:14:30,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=7185, suffix heap size=148296,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=157 2024-11-12T18:14:30,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731435270145, max timestamp=1731435270161Num uniques 1166; 2024-11-12T18:14:30,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53396, new segment data size=7596, suffix heap size=149232,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1166, new segment cells count=166 2024-11-12T18:14:30,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731435270145, max timestamp=1731435270161Num uniques 1016; 2024-11-12T18:14:30,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=7596, suffix heap size=133632,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=166 2024-11-12T18:14:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435330161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731435270145, max timestamp=1731435270161Num uniques 1166; 2024-11-12T18:14:30,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53396, new segment data size=7596, suffix heap size=149232,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1166, new segment cells count=166 2024-11-12T18:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1731435270213, max timestamp=1731435270218Num uniques 1025; 2024-11-12T18:14:30,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1731435270213, max timestamp=1731435270218Num uniques 1025; 2024-11-12T18:14:30,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=14880, suffix heap size=134568,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=325 2024-11-12T18:14:30,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=14880, suffix heap size=134568,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=325 2024-11-12T18:14:30,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=325, cellSize=14880, totalHeapSize=34040, min timestamp=1731435270213, max timestamp=1731435270218Num uniques 1025; 2024-11-12T18:14:30,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=14880, suffix heap size=134568,new segment heap size=34040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=325 2024-11-12T18:14:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1731435270242, max timestamp=1731435270254Num uniques 1184; 2024-11-12T18:14:30,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54224, new segment data size=8424, suffix heap size=151104,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1184, new segment cells count=184 2024-11-12T18:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435330257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1731435270242, max timestamp=1731435270254Num uniques 1184; 2024-11-12T18:14:30,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54224, new segment data size=8424, suffix heap size=151104,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1184, new segment cells count=184 2024-11-12T18:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=184, cellSize=8424, totalHeapSize=19376, min timestamp=1731435270242, max timestamp=1731435270254Num uniques 1184; 2024-11-12T18:14:30,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54224, new segment data size=8424, suffix heap size=151104,new segment heap size=19376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1184, new segment cells count=184 2024-11-12T18:14:30,268 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/6b6f846dfe074a7aa79cb3b90da539e2 2024-11-12T18:14:30,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/ced8c53db9284678b12cca2d44d4ec31 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31 2024-11-12T18:14:30,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31, entries=100, sequenceid=292, filesize=22.0 K 2024-11-12T18:14:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/aae86d7a11d64ef693c31354f09ccee4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/aae86d7a11d64ef693c31354f09ccee4 2024-11-12T18:14:30,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/aae86d7a11d64ef693c31354f09ccee4, entries=100, sequenceid=292, filesize=9.6 K 2024-11-12T18:14:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/6b6f846dfe074a7aa79cb3b90da539e2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6b6f846dfe074a7aa79cb3b90da539e2 2024-11-12T18:14:30,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=343, cellSize=15708, totalHeapSize=35912, min timestamp=1731435270269, max timestamp=1731435270289Num uniques 1043; 2024-11-12T18:14:30,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=15708, suffix heap size=136440,new segment heap size=35912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=343 2024-11-12T18:14:30,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=343, cellSize=15708, totalHeapSize=35912, min timestamp=1731435270269, max timestamp=1731435270289Num uniques 1043; 2024-11-12T18:14:30,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=15708, suffix heap size=136440,new segment heap size=35912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=343 2024-11-12T18:14:30,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=343, cellSize=15708, totalHeapSize=35912, min timestamp=1731435270269, max timestamp=1731435270289Num uniques 1043; 2024-11-12T18:14:30,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47768, new segment data size=15708, suffix heap size=136440,new segment heap size=35912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1043, new segment cells count=343 2024-11-12T18:14:30,299 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6b6f846dfe074a7aa79cb3b90da539e2, entries=100, sequenceid=292, filesize=9.6 K 2024-11-12T18:14:30,300 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=73.80 KB/75570 for ae79f5068ec1983ef2be0dd82f4139ba in 1385ms, sequenceid=292, compaction requested=false 2024-11-12T18:14:30,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:30,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-12T18:14:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:30,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=80.51 KB heapSize=208.36 KB 2024-11-12T18:14:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-12T18:14:30,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T18:14:30,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0070 sec 2024-11-12T18:14:30,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e91710327ada4fc8ac2f509387aeca20_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:30,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0150 sec 2024-11-12T18:14:30,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741995_1171 (size=22618) 2024-11-12T18:14:30,326 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:30,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e91710327ada4fc8ac2f509387aeca20_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e91710327ada4fc8ac2f509387aeca20_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:30,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c57783c8bb3f4002a4dcdd23eadd1087, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:30,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c57783c8bb3f4002a4dcdd23eadd1087 is 175, key is test_row_0/A:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:30,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741996_1172 (size=66023) 2024-11-12T18:14:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435330348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435330349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435330349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435330370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435330370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T18:14:30,440 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:30,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-12T18:14:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:30,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:30,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:30,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:30,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435330461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435330470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435330470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:30,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435330580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435330670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435330681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435330690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435330690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,739 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=766, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c57783c8bb3f4002a4dcdd23eadd1087 2024-11-12T18:14:30,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/8270a83346094d67bd037f6398fc6517 is 50, key is test_row_0/B:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:30,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741997_1173 (size=12301) 2024-11-12T18:14:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:30,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435330890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:30,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:30,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:30,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:30,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:30,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435330980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:30,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:30,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435331000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435331000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:31,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/8270a83346094d67bd037f6398fc6517 2024-11-12T18:14:31,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8b393569ff1c495e88ddd0734cd47f96 is 50, key is test_row_0/C:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:31,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741998_1174 (size=12301) 2024-11-12T18:14:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435331190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:31,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:31,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:31,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:31,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435331411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435331490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435331510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435331510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:31,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:31,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8b393569ff1c495e88ddd0734cd47f96 2024-11-12T18:14:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:31,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c57783c8bb3f4002a4dcdd23eadd1087 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087 2024-11-12T18:14:31,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087, entries=350, sequenceid=766, filesize=64.5 K 2024-11-12T18:14:31,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/8270a83346094d67bd037f6398fc6517 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/8270a83346094d67bd037f6398fc6517 2024-11-12T18:14:31,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/8270a83346094d67bd037f6398fc6517, entries=150, sequenceid=766, filesize=12.0 K 2024-11-12T18:14:31,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8b393569ff1c495e88ddd0734cd47f96 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8b393569ff1c495e88ddd0734cd47f96 2024-11-12T18:14:31,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8b393569ff1c495e88ddd0734cd47f96, entries=150, sequenceid=766, filesize=12.0 K 2024-11-12T18:14:31,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~208.31 KB/213312, currentSize=114.05 KB/116790 for ae79f5068ec1983ef2be0dd82f4139ba in 1298ms, sequenceid=766, compaction requested=true 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:31,600 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:31,600 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:31,601 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:31,601 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:31,601 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,602 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e730d1c86c024399bd327d8e7fa7786d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=117.4 K 2024-11-12T18:14:31,602 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,602 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e730d1c86c024399bd327d8e7fa7786d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087] 2024-11-12T18:14:31,602 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:31,602 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:31,602 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,602 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/fab9cb788c044110bf93b45c7756807d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/aae86d7a11d64ef693c31354f09ccee4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/8270a83346094d67bd037f6398fc6517] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=34.0 K 2024-11-12T18:14:31,603 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting fab9cb788c044110bf93b45c7756807d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731435266753 2024-11-12T18:14:31,603 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e730d1c86c024399bd327d8e7fa7786d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731435266753 2024-11-12T18:14:31,603 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting aae86d7a11d64ef693c31354f09ccee4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731435267935 2024-11-12T18:14:31,603 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ced8c53db9284678b12cca2d44d4ec31, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731435267935 2024-11-12T18:14:31,604 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8270a83346094d67bd037f6398fc6517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731435270286 2024-11-12T18:14:31,604 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c57783c8bb3f4002a4dcdd23eadd1087, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731435270286 2024-11-12T18:14:31,613 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:31,616 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411128c1f56aadbdf4c69a8f1598f0991c3a4_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:31,616 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:31,617 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/589a55b4aa444f7aa2277acf400ffaa0 is 50, key is test_row_0/B:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:31,619 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411128c1f56aadbdf4c69a8f1598f0991c3a4_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:31,619 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128c1f56aadbdf4c69a8f1598f0991c3a4_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:31,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741999_1175 (size=12745) 2024-11-12T18:14:31,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742000_1176 (size=4469) 2024-11-12T18:14:31,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:31,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:31,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ac1f60b576ef41dfbe13e68b4ca1150a_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435270341/Put/seqid=0 2024-11-12T18:14:31,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742001_1177 (size=12454) 2024-11-12T18:14:32,026 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#146 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:32,026 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/288a5fc3abae4388a6f2461181474709 is 175, key is test_row_0/A:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:32,030 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/589a55b4aa444f7aa2277acf400ffaa0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/589a55b4aa444f7aa2277acf400ffaa0 2024-11-12T18:14:32,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742002_1178 (size=31699) 2024-11-12T18:14:32,039 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into 589a55b4aa444f7aa2277acf400ffaa0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:32,039 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:32,039 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435271600; duration=0sec 2024-11-12T18:14:32,039 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:32,039 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:32,039 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:32,041 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/288a5fc3abae4388a6f2461181474709 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/288a5fc3abae4388a6f2461181474709 2024-11-12T18:14:32,041 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:32,041 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:32,041 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:32,041 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/be3d5189874a43aaafe4e0c779e44cc1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6b6f846dfe074a7aa79cb3b90da539e2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8b393569ff1c495e88ddd0734cd47f96] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=34.0 K 2024-11-12T18:14:32,041 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting be3d5189874a43aaafe4e0c779e44cc1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731435266753 2024-11-12T18:14:32,042 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6f846dfe074a7aa79cb3b90da539e2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731435267935 2024-11-12T18:14:32,042 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b393569ff1c495e88ddd0734cd47f96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731435270286 2024-11-12T18:14:32,046 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into 288a5fc3abae4388a6f2461181474709(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:32,046 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:32,046 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435271600; duration=0sec 2024-11-12T18:14:32,046 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:32,046 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:32,050 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:32,051 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/d7c218c5b15a40e08d2d597cd64a7565 is 50, key is test_row_0/C:col10/1731435270298/Put/seqid=0 2024-11-12T18:14:32,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742003_1179 (size=12711) 2024-11-12T18:14:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:32,093 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ac1f60b576ef41dfbe13e68b4ca1150a_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ac1f60b576ef41dfbe13e68b4ca1150a_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:32,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5a1625ff7f364299be01c052f3ed39ad, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:32,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5a1625ff7f364299be01c052f3ed39ad is 175, key is test_row_0/A:col10/1731435270341/Put/seqid=0 2024-11-12T18:14:32,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742004_1180 (size=31255) 2024-11-12T18:14:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:32,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:32,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435332237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435332350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435332420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,424 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,462 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/d7c218c5b15a40e08d2d597cd64a7565 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d7c218c5b15a40e08d2d597cd64a7565 2024-11-12T18:14:32,469 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into d7c218c5b15a40e08d2d597cd64a7565(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:32,469 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:32,469 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=13, startTime=1731435271600; duration=0sec 2024-11-12T18:14:32,469 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:32,470 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435332500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,501 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,503 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=786, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5a1625ff7f364299be01c052f3ed39ad 2024-11-12T18:14:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/1d992c7331bc436da14662d7e611d37b is 50, key is test_row_0/B:col10/1731435270341/Put/seqid=0 2024-11-12T18:14:32,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742005_1181 (size=12301) 2024-11-12T18:14:32,517 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/1d992c7331bc436da14662d7e611d37b 2024-11-12T18:14:32,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435332520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,527 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435332531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,533 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/4fba4df0cd93410c84f2f69d8cfdbf07 is 50, key is test_row_0/C:col10/1731435270341/Put/seqid=0 2024-11-12T18:14:32,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742006_1182 (size=12301) 2024-11-12T18:14:32,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435332560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:32,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435332870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:32,947 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/4fba4df0cd93410c84f2f69d8cfdbf07 2024-11-12T18:14:32,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5a1625ff7f364299be01c052f3ed39ad as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad 2024-11-12T18:14:32,964 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad, entries=150, sequenceid=786, filesize=30.5 K 2024-11-12T18:14:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/1d992c7331bc436da14662d7e611d37b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/1d992c7331bc436da14662d7e611d37b 2024-11-12T18:14:32,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/1d992c7331bc436da14662d7e611d37b, entries=150, sequenceid=786, filesize=12.0 K 2024-11-12T18:14:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/4fba4df0cd93410c84f2f69d8cfdbf07 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/4fba4df0cd93410c84f2f69d8cfdbf07 2024-11-12T18:14:32,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/4fba4df0cd93410c84f2f69d8cfdbf07, entries=150, sequenceid=786, filesize=12.0 K 2024-11-12T18:14:32,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae79f5068ec1983ef2be0dd82f4139ba in 1309ms, sequenceid=786, compaction requested=false 2024-11-12T18:14:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-12T18:14:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-12T18:14:32,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T18:14:32,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5400 sec 2024-11-12T18:14:32,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5460 sec 2024-11-12T18:14:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:33,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:33,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:33,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:33,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:33,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:33,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:33,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:33,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123a76773f4cff49a197f64fa71437b7bd_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:33,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742007_1183 (size=14994) 2024-11-12T18:14:33,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:33,432 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123a76773f4cff49a197f64fa71437b7bd_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123a76773f4cff49a197f64fa71437b7bd_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:33,434 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/8a4c7c000f384e11a42f86b4f6f21e05, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:33,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/8a4c7c000f384e11a42f86b4f6f21e05 is 175, key is test_row_0/A:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:33,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742008_1184 (size=39949) 2024-11-12T18:14:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435333494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435333610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435333820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:33,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=802, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/8a4c7c000f384e11a42f86b4f6f21e05 2024-11-12T18:14:33,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/0a31b0672a864f1d9bcc6fff98e18578 is 50, key is test_row_0/B:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:33,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742009_1185 (size=12301) 2024-11-12T18:14:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435334130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:34,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/0a31b0672a864f1d9bcc6fff98e18578 2024-11-12T18:14:34,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/1b6021ee21774f02bb8349be0cdd8f9d is 50, key is test_row_0/C:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:34,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742010_1186 (size=12301) 2024-11-12T18:14:34,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435334440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:34,443 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:34,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435334510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:34,513 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435334541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435334550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:34,551 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:34,552 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T18:14:34,611 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:34,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-12T18:14:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T18:14:34,615 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:34,615 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:34,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:34,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435334640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:34,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/1b6021ee21774f02bb8349be0cdd8f9d 2024-11-12T18:14:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T18:14:34,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/8a4c7c000f384e11a42f86b4f6f21e05 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05 2024-11-12T18:14:34,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05, entries=200, sequenceid=802, filesize=39.0 K 2024-11-12T18:14:34,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/0a31b0672a864f1d9bcc6fff98e18578 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/0a31b0672a864f1d9bcc6fff98e18578 2024-11-12T18:14:34,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/0a31b0672a864f1d9bcc6fff98e18578, entries=150, sequenceid=802, filesize=12.0 K 2024-11-12T18:14:34,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/1b6021ee21774f02bb8349be0cdd8f9d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1b6021ee21774f02bb8349be0cdd8f9d 2024-11-12T18:14:34,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1b6021ee21774f02bb8349be0cdd8f9d, entries=150, sequenceid=802, filesize=12.0 K 2024-11-12T18:14:34,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ae79f5068ec1983ef2be0dd82f4139ba in 1360ms, sequenceid=802, compaction requested=true 2024-11-12T18:14:34,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:34,754 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:34,755 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:34,756 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:34,756 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:34,756 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/288a5fc3abae4388a6f2461181474709, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=100.5 K 2024-11-12T18:14:34,756 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:34,756 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/288a5fc3abae4388a6f2461181474709, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05] 2024-11-12T18:14:34,757 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288a5fc3abae4388a6f2461181474709, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731435270286 2024-11-12T18:14:34,759 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a1625ff7f364299be01c052f3ed39ad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=786, earliestPutTs=1731435270330 2024-11-12T18:14:34,759 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4c7c000f384e11a42f86b4f6f21e05, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731435272205 2024-11-12T18:14:34,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:34,766 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:34,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T18:14:34,768 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:34,768 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:34,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:34,769 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:34,769 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/589a55b4aa444f7aa2277acf400ffaa0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/1d992c7331bc436da14662d7e611d37b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/0a31b0672a864f1d9bcc6fff98e18578] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.5 K 2024-11-12T18:14:34,769 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:34,769 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 589a55b4aa444f7aa2277acf400ffaa0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731435270286 2024-11-12T18:14:34,770 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d992c7331bc436da14662d7e611d37b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=786, earliestPutTs=1731435270330 2024-11-12T18:14:34,772 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a31b0672a864f1d9bcc6fff98e18578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731435272205 2024-11-12T18:14:34,773 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:34,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:34,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:34,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:34,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:34,810 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:34,812 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/e93cd96cdd5142d784a73e9992838dc9 is 50, key is test_row_0/B:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:34,818 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111243e233e0e52c4b0d9e04701b695a66f0_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:34,820 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111243e233e0e52c4b0d9e04701b695a66f0_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:34,820 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111243e233e0e52c4b0d9e04701b695a66f0_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:34,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ed5c3265cf604512ad360c304701af56_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435273491/Put/seqid=0 2024-11-12T18:14:34,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742012_1188 (size=4469) 2024-11-12T18:14:34,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742011_1187 (size=12847) 2024-11-12T18:14:34,895 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/e93cd96cdd5142d784a73e9992838dc9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e93cd96cdd5142d784a73e9992838dc9 2024-11-12T18:14:34,901 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into e93cd96cdd5142d784a73e9992838dc9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:34,902 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:34,902 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435274766; duration=0sec 2024-11-12T18:14:34,902 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:34,902 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:34,902 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:34,914 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:34,914 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:34,914 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:34,914 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d7c218c5b15a40e08d2d597cd64a7565, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/4fba4df0cd93410c84f2f69d8cfdbf07, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1b6021ee21774f02bb8349be0cdd8f9d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.4 K 2024-11-12T18:14:34,915 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c218c5b15a40e08d2d597cd64a7565, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=766, earliestPutTs=1731435270286 2024-11-12T18:14:34,915 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fba4df0cd93410c84f2f69d8cfdbf07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=786, earliestPutTs=1731435270330 2024-11-12T18:14:34,916 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6021ee21774f02bb8349be0cdd8f9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731435272205 2024-11-12T18:14:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742013_1189 (size=12454) 2024-11-12T18:14:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T18:14:34,941 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:34,942 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/0af8bb0c22294b2e8e5b6371cc3e5879 is 50, key is test_row_0/C:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:34,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742014_1190 (size=12813) 2024-11-12T18:14:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T18:14:35,285 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#155 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:35,286 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/ef77e30e3adc4c18b75b66cf57abf820 is 175, key is test_row_0/A:col10/1731435272205/Put/seqid=0 2024-11-12T18:14:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742015_1191 (size=31801) 2024-11-12T18:14:35,314 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/ef77e30e3adc4c18b75b66cf57abf820 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ef77e30e3adc4c18b75b66cf57abf820 2024-11-12T18:14:35,321 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into ef77e30e3adc4c18b75b66cf57abf820(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:35,321 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:35,321 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435274754; duration=0sec 2024-11-12T18:14:35,321 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:35,321 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:35,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:35,329 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ed5c3265cf604512ad360c304701af56_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ed5c3265cf604512ad360c304701af56_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:35,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c2ebc2c42c534d2491b6ce912432008a, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c2ebc2c42c534d2491b6ce912432008a is 175, key is test_row_0/A:col10/1731435273491/Put/seqid=0 2024-11-12T18:14:35,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742016_1192 (size=31255) 2024-11-12T18:14:35,378 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/0af8bb0c22294b2e8e5b6371cc3e5879 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/0af8bb0c22294b2e8e5b6371cc3e5879 2024-11-12T18:14:35,387 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into 0af8bb0c22294b2e8e5b6371cc3e5879(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:35,388 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:35,388 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=13, startTime=1731435274778; duration=0sec 2024-11-12T18:14:35,388 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:35,388 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:35,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:35,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435335717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:35,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=821, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c2ebc2c42c534d2491b6ce912432008a 2024-11-12T18:14:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/a9774d434a4a433d84aece773da8835f is 50, key is test_row_0/B:col10/1731435273491/Put/seqid=0 2024-11-12T18:14:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T18:14:35,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742017_1193 (size=12301) 2024-11-12T18:14:35,779 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=821 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/a9774d434a4a433d84aece773da8835f 2024-11-12T18:14:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/e0af43577b414009a4ab7c623b7fff74 is 50, key is test_row_0/C:col10/1731435273491/Put/seqid=0 2024-11-12T18:14:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435335831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:35,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742018_1194 (size=12301) 2024-11-12T18:14:35,864 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=821 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/e0af43577b414009a4ab7c623b7fff74 2024-11-12T18:14:35,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/c2ebc2c42c534d2491b6ce912432008a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a 2024-11-12T18:14:35,882 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a, entries=150, sequenceid=821, filesize=30.5 K 2024-11-12T18:14:35,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/a9774d434a4a433d84aece773da8835f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a9774d434a4a433d84aece773da8835f 2024-11-12T18:14:35,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a9774d434a4a433d84aece773da8835f, entries=150, sequenceid=821, filesize=12.0 K 2024-11-12T18:14:35,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/e0af43577b414009a4ab7c623b7fff74 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e0af43577b414009a4ab7c623b7fff74 2024-11-12T18:14:35,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e0af43577b414009a4ab7c623b7fff74, entries=150, sequenceid=821, filesize=12.0 K 2024-11-12T18:14:35,899 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ae79f5068ec1983ef2be0dd82f4139ba in 1130ms, sequenceid=821, compaction requested=false 2024-11-12T18:14:35,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:35,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:35,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-12T18:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-12T18:14:35,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T18:14:35,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2850 sec 2024-11-12T18:14:35,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2930 sec 2024-11-12T18:14:36,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:14:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:36,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129e6998b1c595412d82799082424fc809_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:36,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742019_1195 (size=14994) 2024-11-12T18:14:36,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435336148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:36,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435336260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:36,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435336471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:36,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:36,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129e6998b1c595412d82799082424fc809_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e6998b1c595412d82799082424fc809_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:36,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5265325f33fd45518346926fa15120c2, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:36,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5265325f33fd45518346926fa15120c2 is 175, key is test_row_0/A:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:36,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742020_1196 (size=39949) 2024-11-12T18:14:36,542 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=838, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5265325f33fd45518346926fa15120c2 2024-11-12T18:14:36,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/f2bd262d7c3e4089ab6f9817e93056db is 50, key is test_row_0/B:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742021_1197 (size=12301) 2024-11-12T18:14:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T18:14:36,771 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:36,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-12T18:14:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T18:14:36,774 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:36,775 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:36,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:36,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435336790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T18:14:36,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T18:14:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:36,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:37,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=838 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/f2bd262d7c3e4089ab6f9817e93056db 2024-11-12T18:14:37,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/71c5d2be78494e8c902d66ab0b98916c is 50, key is test_row_0/C:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:37,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742022_1198 (size=12301) 2024-11-12T18:14:37,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=838 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/71c5d2be78494e8c902d66ab0b98916c 2024-11-12T18:14:37,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/5265325f33fd45518346926fa15120c2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2 2024-11-12T18:14:37,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2, entries=200, sequenceid=838, filesize=39.0 K 2024-11-12T18:14:37,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/f2bd262d7c3e4089ab6f9817e93056db as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f2bd262d7c3e4089ab6f9817e93056db 2024-11-12T18:14:37,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f2bd262d7c3e4089ab6f9817e93056db, entries=150, sequenceid=838, filesize=12.0 K 2024-11-12T18:14:37,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/71c5d2be78494e8c902d66ab0b98916c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/71c5d2be78494e8c902d66ab0b98916c 2024-11-12T18:14:37,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/71c5d2be78494e8c902d66ab0b98916c, entries=150, sequenceid=838, filesize=12.0 K 2024-11-12T18:14:37,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ae79f5068ec1983ef2be0dd82f4139ba in 1029ms, sequenceid=838, compaction requested=true 2024-11-12T18:14:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:37,076 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:37,076 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:37,078 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:37,078 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:37,078 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:37,078 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:37,078 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:37,078 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:37,078 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e93cd96cdd5142d784a73e9992838dc9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a9774d434a4a433d84aece773da8835f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f2bd262d7c3e4089ab6f9817e93056db] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.6 K 2024-11-12T18:14:37,078 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ef77e30e3adc4c18b75b66cf57abf820, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=100.6 K 2024-11-12T18:14:37,078 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:37,078 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ef77e30e3adc4c18b75b66cf57abf820, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2] 2024-11-12T18:14:37,079 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e93cd96cdd5142d784a73e9992838dc9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731435272205 2024-11-12T18:14:37,079 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef77e30e3adc4c18b75b66cf57abf820, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731435272205 2024-11-12T18:14:37,079 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ebc2c42c534d2491b6ce912432008a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=821, earliestPutTs=1731435273438 2024-11-12T18:14:37,079 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a9774d434a4a433d84aece773da8835f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=821, earliestPutTs=1731435273438 2024-11-12T18:14:37,079 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5265325f33fd45518346926fa15120c2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1731435275687 2024-11-12T18:14:37,080 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f2bd262d7c3e4089ab6f9817e93056db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1731435275707 2024-11-12T18:14:37,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T18:14:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:37,082 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:14:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T18:14:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111221dab2f6555a49efa781cb338950b8a2_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435276139/Put/seqid=0 2024-11-12T18:14:37,103 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:37,104 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/350c9a04176b441baeb3c8633f8b2266 is 50, key is test_row_0/B:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:37,115 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:37,121 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112a5bca36bdfdf4b0989ffaa12facac3dd_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:37,123 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112a5bca36bdfdf4b0989ffaa12facac3dd_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:37,124 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a5bca36bdfdf4b0989ffaa12facac3dd_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:37,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742024_1200 (size=12949) 2024-11-12T18:14:37,145 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/350c9a04176b441baeb3c8633f8b2266 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/350c9a04176b441baeb3c8633f8b2266 2024-11-12T18:14:37,154 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into 350c9a04176b441baeb3c8633f8b2266(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:37,154 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:37,154 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435277076; duration=0sec 2024-11-12T18:14:37,154 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:37,154 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:37,154 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:37,156 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:37,156 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:37,156 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:37,156 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/0af8bb0c22294b2e8e5b6371cc3e5879, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e0af43577b414009a4ab7c623b7fff74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/71c5d2be78494e8c902d66ab0b98916c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.5 K 2024-11-12T18:14:37,156 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af8bb0c22294b2e8e5b6371cc3e5879, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1731435272205 2024-11-12T18:14:37,157 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e0af43577b414009a4ab7c623b7fff74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=821, earliestPutTs=1731435273438 2024-11-12T18:14:37,158 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c5d2be78494e8c902d66ab0b98916c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1731435275707 2024-11-12T18:14:37,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742023_1199 (size=12454) 2024-11-12T18:14:37,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:37,173 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111221dab2f6555a49efa781cb338950b8a2_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111221dab2f6555a49efa781cb338950b8a2_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9803b516f9fd480eb03f9506d29f4bc7, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:37,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9803b516f9fd480eb03f9506d29f4bc7 is 175, key is test_row_0/A:col10/1731435276139/Put/seqid=0 2024-11-12T18:14:37,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742025_1201 (size=4469) 2024-11-12T18:14:37,193 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#166 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:37,193 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/12df7b08562f4fff962f548ad4ec7708 is 175, key is test_row_0/A:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:37,207 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:37,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742026_1202 (size=31255) 2024-11-12T18:14:37,207 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8786a16d45f4453494f0de512ac7fe7e is 50, key is test_row_0/C:col10/1731435276045/Put/seqid=0 2024-11-12T18:14:37,208 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=856, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9803b516f9fd480eb03f9506d29f4bc7 2024-11-12T18:14:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/d788e6e8c399444bb52b5d8799b07398 is 50, key is test_row_0/B:col10/1731435276139/Put/seqid=0 2024-11-12T18:14:37,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742027_1203 (size=31903) 2024-11-12T18:14:37,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742028_1204 (size=12915) 2024-11-12T18:14:37,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742029_1205 (size=12301) 2024-11-12T18:14:37,248 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/d788e6e8c399444bb52b5d8799b07398 2024-11-12T18:14:37,252 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/8786a16d45f4453494f0de512ac7fe7e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8786a16d45f4453494f0de512ac7fe7e 2024-11-12T18:14:37,262 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into 8786a16d45f4453494f0de512ac7fe7e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:37,262 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:37,262 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=13, startTime=1731435277077; duration=0sec 2024-11-12T18:14:37,262 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:37,262 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:37,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c9a3203fd904480ca42b2ab36b10cd8b is 50, key is test_row_0/C:col10/1731435276139/Put/seqid=0 2024-11-12T18:14:37,264 INFO [master/9911683f163c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T18:14:37,264 INFO [master/9911683f163c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T18:14:37,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742030_1206 (size=12301) 2024-11-12T18:14:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:37,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435337358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T18:14:37,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435337470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:37,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:37,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:37,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:37,644 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/12df7b08562f4fff962f548ad4ec7708 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/12df7b08562f4fff962f548ad4ec7708 2024-11-12T18:14:37,656 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into 12df7b08562f4fff962f548ad4ec7708(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:37,656 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:37,656 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435277076; duration=0sec 2024-11-12T18:14:37,656 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:37,656 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:37,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c9a3203fd904480ca42b2ab36b10cd8b 2024-11-12T18:14:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/9803b516f9fd480eb03f9506d29f4bc7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7 2024-11-12T18:14:37,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435337681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:37,696 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7, entries=150, sequenceid=856, filesize=30.5 K 2024-11-12T18:14:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/d788e6e8c399444bb52b5d8799b07398 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/d788e6e8c399444bb52b5d8799b07398 2024-11-12T18:14:37,707 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/d788e6e8c399444bb52b5d8799b07398, entries=150, sequenceid=856, filesize=12.0 K 2024-11-12T18:14:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c9a3203fd904480ca42b2ab36b10cd8b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c9a3203fd904480ca42b2ab36b10cd8b 2024-11-12T18:14:37,716 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c9a3203fd904480ca42b2ab36b10cd8b, entries=150, sequenceid=856, filesize=12.0 K 2024-11-12T18:14:37,717 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ae79f5068ec1983ef2be0dd82f4139ba in 635ms, sequenceid=856, compaction requested=false 2024-11-12T18:14:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T18:14:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T18:14:37,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T18:14:37,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 944 msec 2024-11-12T18:14:37,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 950 msec 2024-11-12T18:14:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T18:14:37,910 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:37,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-12T18:14:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:37,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:37,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:37,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:37,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:38,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111231a687282cbe4b7c9212272e523921d1_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742031_1207 (size=14994) 2024-11-12T18:14:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:38,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:38,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:38,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435338068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435338180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:38,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:38,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:38,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435338390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:38,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111231a687282cbe4b7c9212272e523921d1_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111231a687282cbe4b7c9212272e523921d1_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:38,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/91e4f784d5db41e2b4e041b71f468e49, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:38,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/91e4f784d5db41e2b4e041b71f468e49 is 175, key is test_row_0/A:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:38,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742032_1208 (size=39949) 2024-11-12T18:14:38,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1731435338460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,470 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:38,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38898 deadline: 1731435338541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,542 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38928 deadline: 1731435338570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,576 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38926 deadline: 1731435338580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,583 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ae79f5068ec1983ef2be0dd82f4139ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:38,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:38,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:38,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435338701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:38,840 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=874, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/91e4f784d5db41e2b4e041b71f468e49 2024-11-12T18:14:38,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/64c0993e1f034791880794757a6e414d is 50, key is test_row_0/B:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:38,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:38,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:38,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742033_1209 (size=12301) 2024-11-12T18:14:39,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:39,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:39,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435339211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:39,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/64c0993e1f034791880794757a6e414d 2024-11-12T18:14:39,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/887d0cb2b8344fb1aeaac051521c9ea7 is 50, key is test_row_0/C:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:39,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742034_1210 (size=12301) 2024-11-12T18:14:39,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/887d0cb2b8344fb1aeaac051521c9ea7 2024-11-12T18:14:39,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:39,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/91e4f784d5db41e2b4e041b71f468e49 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49 2024-11-12T18:14:39,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49, entries=200, sequenceid=874, filesize=39.0 K 2024-11-12T18:14:39,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/64c0993e1f034791880794757a6e414d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/64c0993e1f034791880794757a6e414d 2024-11-12T18:14:39,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/64c0993e1f034791880794757a6e414d, entries=150, sequenceid=874, filesize=12.0 K 2024-11-12T18:14:39,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/887d0cb2b8344fb1aeaac051521c9ea7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/887d0cb2b8344fb1aeaac051521c9ea7 2024-11-12T18:14:39,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/887d0cb2b8344fb1aeaac051521c9ea7, entries=150, sequenceid=874, filesize=12.0 K 2024-11-12T18:14:39,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ae79f5068ec1983ef2be0dd82f4139ba in 1360ms, sequenceid=874, compaction requested=true 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:39,353 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:39,353 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:39,358 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:39,358 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:39,358 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,358 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/350c9a04176b441baeb3c8633f8b2266, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/d788e6e8c399444bb52b5d8799b07398, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/64c0993e1f034791880794757a6e414d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.7 K 2024-11-12T18:14:39,358 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:39,358 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:39,358 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,358 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/12df7b08562f4fff962f548ad4ec7708, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=100.7 K 2024-11-12T18:14:39,358 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,358 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/12df7b08562f4fff962f548ad4ec7708, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49] 2024-11-12T18:14:39,359 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 350c9a04176b441baeb3c8633f8b2266, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1731435275707 2024-11-12T18:14:39,359 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12df7b08562f4fff962f548ad4ec7708, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1731435275707 2024-11-12T18:14:39,360 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d788e6e8c399444bb52b5d8799b07398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731435276139 2024-11-12T18:14:39,360 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9803b516f9fd480eb03f9506d29f4bc7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731435276139 2024-11-12T18:14:39,360 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 64c0993e1f034791880794757a6e414d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731435277337 2024-11-12T18:14:39,360 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91e4f784d5db41e2b4e041b71f468e49, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731435277337 2024-11-12T18:14:39,380 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:39,387 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#173 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:39,387 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/9f9bab04470d40558d3f4fb30ebfcccc is 50, key is test_row_0/B:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:39,395 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111227e0329b58df4aedbfd9c2c7e6581aa6_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:39,397 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111227e0329b58df4aedbfd9c2c7e6581aa6_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:39,397 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111227e0329b58df4aedbfd9c2c7e6581aa6_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:39,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742035_1211 (size=13051) 2024-11-12T18:14:39,429 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/9f9bab04470d40558d3f4fb30ebfcccc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/9f9bab04470d40558d3f4fb30ebfcccc 2024-11-12T18:14:39,435 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into 9f9bab04470d40558d3f4fb30ebfcccc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:39,435 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:39,435 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435279353; duration=0sec 2024-11-12T18:14:39,435 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:39,436 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:39,436 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:39,439 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:39,439 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:39,439 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,439 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8786a16d45f4453494f0de512ac7fe7e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c9a3203fd904480ca42b2ab36b10cd8b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/887d0cb2b8344fb1aeaac051521c9ea7] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.6 K 2024-11-12T18:14:39,440 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8786a16d45f4453494f0de512ac7fe7e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=838, earliestPutTs=1731435275707 2024-11-12T18:14:39,440 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a3203fd904480ca42b2ab36b10cd8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731435276139 2024-11-12T18:14:39,443 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 887d0cb2b8344fb1aeaac051521c9ea7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731435277337 2024-11-12T18:14:39,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742036_1212 (size=4469) 2024-11-12T18:14:39,459 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:39,460 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/9af6c65f31c64195848e99397de953ae is 50, key is test_row_0/C:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:39,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:39,471 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:39,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742037_1213 (size=13017) 2024-11-12T18:14:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bf700979a75c45adadc0f6c6683051f5_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435278023/Put/seqid=0 2024-11-12T18:14:39,502 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/9af6c65f31c64195848e99397de953ae as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9af6c65f31c64195848e99397de953ae 2024-11-12T18:14:39,509 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into 9af6c65f31c64195848e99397de953ae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:39,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:39,509 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=13, startTime=1731435279353; duration=0sec 2024-11-12T18:14:39,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:39,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:39,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742038_1214 (size=12454) 2024-11-12T18:14:39,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:39,537 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bf700979a75c45adadc0f6c6683051f5_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bf700979a75c45adadc0f6c6683051f5_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/cfa54f2c62d948c4a13d62fc0372b98f, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:39,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/cfa54f2c62d948c4a13d62fc0372b98f is 175, key is test_row_0/A:col10/1731435278023/Put/seqid=0 2024-11-12T18:14:39,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742039_1215 (size=31255) 2024-11-12T18:14:39,572 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=892, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/cfa54f2c62d948c4a13d62fc0372b98f 2024-11-12T18:14:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/a06781596960471f9cf26e02976e4dd6 is 50, key is test_row_0/B:col10/1731435278023/Put/seqid=0 2024-11-12T18:14:39,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742040_1216 (size=12301) 2024-11-12T18:14:39,857 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#174 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:39,858 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/152da823f119452095671601a4f1641a is 175, key is test_row_0/A:col10/1731435277991/Put/seqid=0 2024-11-12T18:14:39,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742041_1217 (size=32005) 2024-11-12T18:14:39,921 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/152da823f119452095671601a4f1641a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/152da823f119452095671601a4f1641a 2024-11-12T18:14:39,929 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into 152da823f119452095671601a4f1641a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:39,929 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:39,929 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435279353; duration=0sec 2024-11-12T18:14:39,929 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:39,929 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:40,046 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=892 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/a06781596960471f9cf26e02976e4dd6 2024-11-12T18:14:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:40,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/af397aed58ee48b181680b0e3306588e is 50, key is test_row_0/C:col10/1731435278023/Put/seqid=0 2024-11-12T18:14:40,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742042_1218 (size=12301) 2024-11-12T18:14:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:40,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:40,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435340340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435340466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:40,539 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=892 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/af397aed58ee48b181680b0e3306588e 2024-11-12T18:14:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/cfa54f2c62d948c4a13d62fc0372b98f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f 2024-11-12T18:14:40,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f, entries=150, sequenceid=892, filesize=30.5 K 2024-11-12T18:14:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/a06781596960471f9cf26e02976e4dd6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a06781596960471f9cf26e02976e4dd6 2024-11-12T18:14:40,559 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a06781596960471f9cf26e02976e4dd6, entries=150, sequenceid=892, filesize=12.0 K 2024-11-12T18:14:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/af397aed58ee48b181680b0e3306588e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/af397aed58ee48b181680b0e3306588e 2024-11-12T18:14:40,591 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/af397aed58ee48b181680b0e3306588e, entries=150, sequenceid=892, filesize=12.0 K 2024-11-12T18:14:40,596 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae79f5068ec1983ef2be0dd82f4139ba in 1125ms, sequenceid=892, compaction requested=false 2024-11-12T18:14:40,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:40,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:40,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T18:14:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T18:14:40,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T18:14:40,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6850 sec 2024-11-12T18:14:40,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6910 sec 2024-11-12T18:14:40,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T18:14:40,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:40,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:40,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111206522552e7314d7f94c7459cbf1e70d3_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:40,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742043_1219 (size=12454) 2024-11-12T18:14:40,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435340810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435340930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435341141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:41,149 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:41,154 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111206522552e7314d7f94c7459cbf1e70d3_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111206522552e7314d7f94c7459cbf1e70d3_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:41,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/883f88407b25458c8f8c93eb965ce73e, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:41,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/883f88407b25458c8f8c93eb965ce73e is 175, key is test_row_0/A:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:41,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742044_1220 (size=31255) 2024-11-12T18:14:41,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=910, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/883f88407b25458c8f8c93eb965ce73e 2024-11-12T18:14:41,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/63b257b41eca449ebae1cab0a7988d67 is 50, key is test_row_0/B:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:41,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742045_1221 (size=12301) 2024-11-12T18:14:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435341450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:41,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=910 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/63b257b41eca449ebae1cab0a7988d67 2024-11-12T18:14:41,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/523700452e3448329c1b373806646fa3 is 50, key is test_row_0/C:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:41,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742046_1222 (size=12301) 2024-11-12T18:14:41,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=910 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/523700452e3448329c1b373806646fa3 2024-11-12T18:14:41,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/883f88407b25458c8f8c93eb965ce73e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e 2024-11-12T18:14:41,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e, entries=150, sequenceid=910, filesize=30.5 K 2024-11-12T18:14:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/63b257b41eca449ebae1cab0a7988d67 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/63b257b41eca449ebae1cab0a7988d67 2024-11-12T18:14:41,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/63b257b41eca449ebae1cab0a7988d67, entries=150, sequenceid=910, filesize=12.0 K 2024-11-12T18:14:41,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/523700452e3448329c1b373806646fa3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/523700452e3448329c1b373806646fa3 2024-11-12T18:14:41,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/523700452e3448329c1b373806646fa3, entries=150, sequenceid=910, filesize=12.0 K 2024-11-12T18:14:41,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae79f5068ec1983ef2be0dd82f4139ba in 980ms, sequenceid=910, compaction requested=true 2024-11-12T18:14:41,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:41,664 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:41,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:41,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:41,665 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:41,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:41,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:41,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae79f5068ec1983ef2be0dd82f4139ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:41,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:41,666 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:41,666 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/A is initiating minor compaction (all files) 2024-11-12T18:14:41,666 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/A in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:41,666 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:41,666 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/B is initiating minor compaction (all files) 2024-11-12T18:14:41,666 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/152da823f119452095671601a4f1641a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=92.3 K 2024-11-12T18:14:41,666 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/B in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:41,666 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:41,666 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/152da823f119452095671601a4f1641a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e] 2024-11-12T18:14:41,666 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/9f9bab04470d40558d3f4fb30ebfcccc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a06781596960471f9cf26e02976e4dd6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/63b257b41eca449ebae1cab0a7988d67] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.8 K 2024-11-12T18:14:41,667 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152da823f119452095671601a4f1641a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731435277337 2024-11-12T18:14:41,667 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f9bab04470d40558d3f4fb30ebfcccc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731435277337 2024-11-12T18:14:41,667 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a06781596960471f9cf26e02976e4dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=892, earliestPutTs=1731435278023 2024-11-12T18:14:41,667 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfa54f2c62d948c4a13d62fc0372b98f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=892, earliestPutTs=1731435278023 2024-11-12T18:14:41,669 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 63b257b41eca449ebae1cab0a7988d67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=910, earliestPutTs=1731435280322 2024-11-12T18:14:41,669 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 883f88407b25458c8f8c93eb965ce73e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=910, earliestPutTs=1731435280322 2024-11-12T18:14:41,691 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:41,692 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/4c1467b6315840aa902a1a5bdd6943be is 50, key is test_row_0/B:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:41,692 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:41,697 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111224f43a13c2e64ea68b098f9ba14004bf_ae79f5068ec1983ef2be0dd82f4139ba store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:41,699 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111224f43a13c2e64ea68b098f9ba14004bf_ae79f5068ec1983ef2be0dd82f4139ba, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:41,699 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111224f43a13c2e64ea68b098f9ba14004bf_ae79f5068ec1983ef2be0dd82f4139ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742048_1224 (size=4469) 2024-11-12T18:14:41,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742047_1223 (size=13153) 2024-11-12T18:14:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:41,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T18:14:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:41,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:41,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ecdc38cb961949d399c272b6bc02ebaa_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435281966/Put/seqid=0 2024-11-12T18:14:41,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742049_1225 (size=14994) 2024-11-12T18:14:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435342031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T18:14:42,091 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:42,091 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-12T18:14:42,093 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:42,095 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:42,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:42,134 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:42,135 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/282a9515d70447c0b34e83ffb8882905 is 175, key is test_row_0/A:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435342140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:42,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742050_1226 (size=32107) 2024-11-12T18:14:42,153 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/4c1467b6315840aa902a1a5bdd6943be as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/4c1467b6315840aa902a1a5bdd6943be 2024-11-12T18:14:42,154 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/282a9515d70447c0b34e83ffb8882905 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/282a9515d70447c0b34e83ffb8882905 2024-11-12T18:14:42,161 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/B of ae79f5068ec1983ef2be0dd82f4139ba into 4c1467b6315840aa902a1a5bdd6943be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:42,161 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:42,161 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/B, priority=13, startTime=1731435281665; duration=0sec 2024-11-12T18:14:42,161 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:42,161 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:B 2024-11-12T18:14:42,161 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:42,163 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:42,163 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ae79f5068ec1983ef2be0dd82f4139ba/C is initiating minor compaction (all files) 2024-11-12T18:14:42,163 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae79f5068ec1983ef2be0dd82f4139ba/C in TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,163 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9af6c65f31c64195848e99397de953ae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/af397aed58ee48b181680b0e3306588e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/523700452e3448329c1b373806646fa3] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp, totalSize=36.7 K 2024-11-12T18:14:42,163 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af6c65f31c64195848e99397de953ae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=874, earliestPutTs=1731435277337 2024-11-12T18:14:42,164 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting af397aed58ee48b181680b0e3306588e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=892, earliestPutTs=1731435278023 2024-11-12T18:14:42,164 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/A of ae79f5068ec1983ef2be0dd82f4139ba into 282a9515d70447c0b34e83ffb8882905(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:42,164 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 523700452e3448329c1b373806646fa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=910, earliestPutTs=1731435280322 2024-11-12T18:14:42,164 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:42,164 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/A, priority=13, startTime=1731435281664; duration=0sec 2024-11-12T18:14:42,165 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:42,165 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:A 2024-11-12T18:14:42,178 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae79f5068ec1983ef2be0dd82f4139ba#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:42,179 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/e9ff3c94830c4684be7e57e8f24d53c8 is 50, key is test_row_0/C:col10/1731435280337/Put/seqid=0 2024-11-12T18:14:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:42,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742051_1227 (size=13119) 2024-11-12T18:14:42,214 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/e9ff3c94830c4684be7e57e8f24d53c8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e9ff3c94830c4684be7e57e8f24d53c8 2024-11-12T18:14:42,222 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae79f5068ec1983ef2be0dd82f4139ba/C of ae79f5068ec1983ef2be0dd82f4139ba into e9ff3c94830c4684be7e57e8f24d53c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:42,222 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:42,222 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., storeName=ae79f5068ec1983ef2be0dd82f4139ba/C, priority=13, startTime=1731435281665; duration=0sec 2024-11-12T18:14:42,222 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:42,222 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae79f5068ec1983ef2be0dd82f4139ba:C 2024-11-12T18:14:42,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T18:14:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435342350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:42,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:42,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T18:14:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ecdc38cb961949d399c272b6bc02ebaa_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ecdc38cb961949d399c272b6bc02ebaa_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,404 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3bb28d2aed914e81b29d2f7c1d0c0aa9, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:42,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3bb28d2aed914e81b29d2f7c1d0c0aa9 is 175, key is test_row_0/A:col10/1731435281966/Put/seqid=0 2024-11-12T18:14:42,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742052_1228 (size=39949) 2024-11-12T18:14:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:42,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=927, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3bb28d2aed914e81b29d2f7c1d0c0aa9 2024-11-12T18:14:42,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c1e609cebf2a4e90a5585b034d3942b5 is 50, key is test_row_0/B:col10/1731435281966/Put/seqid=0 2024-11-12T18:14:42,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742053_1229 (size=12301) 2024-11-12T18:14:42,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=927 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c1e609cebf2a4e90a5585b034d3942b5 2024-11-12T18:14:42,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/9cfa8e56a8df4e3eadcf1257c01f3cbf is 50, key is test_row_0/C:col10/1731435281966/Put/seqid=0 2024-11-12T18:14:42,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742054_1230 (size=12301) 2024-11-12T18:14:42,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T18:14:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38878 deadline: 1731435342660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:14:42,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T18:14:42,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:42,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:42,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T18:14:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:42,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:42,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=927 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/9cfa8e56a8df4e3eadcf1257c01f3cbf 2024-11-12T18:14:42,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3bb28d2aed914e81b29d2f7c1d0c0aa9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3bb28d2aed914e81b29d2f7c1d0c0aa9 2024-11-12T18:14:42,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3bb28d2aed914e81b29d2f7c1d0c0aa9, entries=200, sequenceid=927, filesize=39.0 K 2024-11-12T18:14:42,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/c1e609cebf2a4e90a5585b034d3942b5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c1e609cebf2a4e90a5585b034d3942b5 2024-11-12T18:14:42,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c1e609cebf2a4e90a5585b034d3942b5, entries=150, sequenceid=927, filesize=12.0 K 2024-11-12T18:14:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/9cfa8e56a8df4e3eadcf1257c01f3cbf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9cfa8e56a8df4e3eadcf1257c01f3cbf 2024-11-12T18:14:42,898 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-12T18:14:42,898 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:42,898 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,899 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:42,901 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-12T18:14:42,901 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:42,902 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,902 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:42,902 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-12T18:14:42,902 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:42,902 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,902 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:42,906 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-12T18:14:42,906 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:42,906 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,907 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:42,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:42,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9cfa8e56a8df4e3eadcf1257c01f3cbf, entries=150, sequenceid=927, filesize=12.0 K 2024-11-12T18:14:42,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae79f5068ec1983ef2be0dd82f4139ba in 939ms, sequenceid=927, compaction requested=false 2024-11-12T18:14:42,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:43,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T18:14:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:43,019 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:14:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:43,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112284e78d0f81b4a70a8fa89df6d856ad7_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435282029/Put/seqid=0 2024-11-12T18:14:43,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742055_1231 (size=12454) 2024-11-12T18:14:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. as already flushing 2024-11-12T18:14:43,172 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-12T18:14:43,172 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:43,172 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:43,173 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:43,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:43,436 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112284e78d0f81b4a70a8fa89df6d856ad7_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112284e78d0f81b4a70a8fa89df6d856ad7_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:43,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3d7cf014f25043ef8d2a788faad981f4, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:43,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3d7cf014f25043ef8d2a788faad981f4 is 175, key is test_row_0/A:col10/1731435282029/Put/seqid=0 2024-11-12T18:14:43,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742056_1232 (size=31255) 2024-11-12T18:14:43,843 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=945, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3d7cf014f25043ef8d2a788faad981f4 2024-11-12T18:14:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/e9c2a94f13234b3e827f742d613af403 is 50, key is test_row_0/B:col10/1731435282029/Put/seqid=0 2024-11-12T18:14:43,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742057_1233 (size=12301) 2024-11-12T18:14:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:44,272 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=945 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/e9c2a94f13234b3e827f742d613af403 2024-11-12T18:14:44,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/60f7cb53c20547de87326750c1d1e502 is 50, key is test_row_0/C:col10/1731435282029/Put/seqid=0 2024-11-12T18:14:44,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742058_1234 (size=12301) 2024-11-12T18:14:44,289 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=945 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/60f7cb53c20547de87326750c1d1e502 2024-11-12T18:14:44,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/3d7cf014f25043ef8d2a788faad981f4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3d7cf014f25043ef8d2a788faad981f4 2024-11-12T18:14:44,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3d7cf014f25043ef8d2a788faad981f4, entries=150, sequenceid=945, filesize=30.5 K 2024-11-12T18:14:44,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/e9c2a94f13234b3e827f742d613af403 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e9c2a94f13234b3e827f742d613af403 2024-11-12T18:14:44,307 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e9c2a94f13234b3e827f742d613af403, entries=150, sequenceid=945, filesize=12.0 K 2024-11-12T18:14:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/60f7cb53c20547de87326750c1d1e502 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/60f7cb53c20547de87326750c1d1e502 2024-11-12T18:14:44,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/60f7cb53c20547de87326750c1d1e502, entries=150, sequenceid=945, filesize=12.0 K 2024-11-12T18:14:44,315 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for ae79f5068ec1983ef2be0dd82f4139ba in 1297ms, sequenceid=945, compaction requested=true 2024-11-12T18:14:44,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ae79f5068ec1983ef2be0dd82f4139ba: 2024-11-12T18:14:44,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:44,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-12T18:14:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-12T18:14:44,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T18:14:44,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2220 sec 2024-11-12T18:14:44,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2280 sec 2024-11-12T18:14:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T18:14:46,251 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:48,542 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-11-12T18:14:48,542 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:48,542 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,542 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:48,572 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-12T18:14:48,572 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:48,572 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,572 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:48,592 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-12T18:14:48,592 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:48,592 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,592 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:48,652 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-12T18:14:48,652 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:14:48,652 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,652 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:48,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 181 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 272 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2520 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2461 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3380 rows 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1126 2024-11-12T18:14:48,653 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3374 rows 2024-11-12T18:14:48,653 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:14:48,653 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:14:48,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:48,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:14:48,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T18:14:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T18:14:48,659 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435288659"}]},"ts":"1731435288659"} 2024-11-12T18:14:48,661 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T18:14:48,661 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T18:14:48,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:14:48,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, UNASSIGN}] 2024-11-12T18:14:48,664 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, UNASSIGN 2024-11-12T18:14:48,666 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:48,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, UNASSIGN because future has completed 2024-11-12T18:14:48,668 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:14:48,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:14:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T18:14:48,820 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:48,820 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing ae79f5068ec1983ef2be0dd82f4139ba, disabling compactions & flushes 2024-11-12T18:14:48,821 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. after waiting 0 ms 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:48,821 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing ae79f5068ec1983ef2be0dd82f4139ba 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=A 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=B 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae79f5068ec1983ef2be0dd82f4139ba, store=C 2024-11-12T18:14:48,821 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:48,827 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112394aa2f714c9410ab930a21509e012f9_ae79f5068ec1983ef2be0dd82f4139ba is 50, key is test_row_0/A:col10/1731435288590/Put/seqid=0 2024-11-12T18:14:48,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742059_1235 (size=12454) 2024-11-12T18:14:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T18:14:49,232 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:49,237 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112394aa2f714c9410ab930a21509e012f9_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112394aa2f714c9410ab930a21509e012f9_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:49,238 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/0199ea092c5f40259cf0440f98777a90, store: [table=TestAcidGuarantees family=A region=ae79f5068ec1983ef2be0dd82f4139ba] 2024-11-12T18:14:49,238 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/0199ea092c5f40259cf0440f98777a90 is 175, key is test_row_0/A:col10/1731435288590/Put/seqid=0 2024-11-12T18:14:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742060_1236 (size=31255) 2024-11-12T18:14:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T18:14:49,643 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=953, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/0199ea092c5f40259cf0440f98777a90 2024-11-12T18:14:49,650 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/2e0758e74a02419dbde28077264a8bd5 is 50, key is test_row_0/B:col10/1731435288590/Put/seqid=0 2024-11-12T18:14:49,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742061_1237 (size=12301) 2024-11-12T18:14:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T18:14:50,055 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=953 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/2e0758e74a02419dbde28077264a8bd5 2024-11-12T18:14:50,065 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c4a270dc5a224a249a6ec081dc3974c4 is 50, key is test_row_0/C:col10/1731435288590/Put/seqid=0 2024-11-12T18:14:50,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742062_1238 (size=12301) 2024-11-12T18:14:50,079 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=953 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c4a270dc5a224a249a6ec081dc3974c4 2024-11-12T18:14:50,085 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/A/0199ea092c5f40259cf0440f98777a90 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/0199ea092c5f40259cf0440f98777a90 2024-11-12T18:14:50,092 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/0199ea092c5f40259cf0440f98777a90, entries=150, sequenceid=953, filesize=30.5 K 2024-11-12T18:14:50,093 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/B/2e0758e74a02419dbde28077264a8bd5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/2e0758e74a02419dbde28077264a8bd5 2024-11-12T18:14:50,106 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/2e0758e74a02419dbde28077264a8bd5, entries=150, sequenceid=953, filesize=12.0 K 2024-11-12T18:14:50,108 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/.tmp/C/c4a270dc5a224a249a6ec081dc3974c4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c4a270dc5a224a249a6ec081dc3974c4 2024-11-12T18:14:50,119 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c4a270dc5a224a249a6ec081dc3974c4, entries=150, sequenceid=953, filesize=12.0 K 2024-11-12T18:14:50,121 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae79f5068ec1983ef2be0dd82f4139ba in 1300ms, sequenceid=953, compaction requested=true 2024-11-12T18:14:50,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fcf2de00af6f4748a1b498078fe30658, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e4ff9a8b7e994851ab58ea3258fb04dd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e730d1c86c024399bd327d8e7fa7786d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/288a5fc3abae4388a6f2461181474709, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ef77e30e3adc4c18b75b66cf57abf820, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/12df7b08562f4fff962f548ad4ec7708, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/152da823f119452095671601a4f1641a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e] to archive 2024-11-12T18:14:50,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:14:50,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/eb1377fa04a84449b875c290e339c6ab 2024-11-12T18:14:50,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/7a5edef979c347a583cf912960486f93 2024-11-12T18:14:50,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/36cb8ca0c0074578a230ac60a0d9574e 2024-11-12T18:14:50,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fcf2de00af6f4748a1b498078fe30658 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fcf2de00af6f4748a1b498078fe30658 2024-11-12T18:14:50,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/be53f6b188a344038b849b8a7bd368e7 2024-11-12T18:14:50,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9343cfa5febd41e4bcd94357a54e466f 2024-11-12T18:14:50,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/fbdfc8d748db43daae3a699463ad272a 2024-11-12T18:14:50,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e4ff9a8b7e994851ab58ea3258fb04dd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e4ff9a8b7e994851ab58ea3258fb04dd 2024-11-12T18:14:50,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/d0e147150c7b42c984be3faffb8e7d91 2024-11-12T18:14:50,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/85d30712cbed42fc855b6eaf7c9ea593 2024-11-12T18:14:50,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e730d1c86c024399bd327d8e7fa7786d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/e730d1c86c024399bd327d8e7fa7786d 2024-11-12T18:14:50,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ced8c53db9284678b12cca2d44d4ec31 2024-11-12T18:14:50,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c57783c8bb3f4002a4dcdd23eadd1087 2024-11-12T18:14:50,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/288a5fc3abae4388a6f2461181474709 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/288a5fc3abae4388a6f2461181474709 2024-11-12T18:14:50,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5a1625ff7f364299be01c052f3ed39ad 2024-11-12T18:14:50,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/8a4c7c000f384e11a42f86b4f6f21e05 2024-11-12T18:14:50,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ef77e30e3adc4c18b75b66cf57abf820 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/ef77e30e3adc4c18b75b66cf57abf820 2024-11-12T18:14:50,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/c2ebc2c42c534d2491b6ce912432008a 2024-11-12T18:14:50,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/5265325f33fd45518346926fa15120c2 2024-11-12T18:14:50,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/12df7b08562f4fff962f548ad4ec7708 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/12df7b08562f4fff962f548ad4ec7708 2024-11-12T18:14:50,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/9803b516f9fd480eb03f9506d29f4bc7 2024-11-12T18:14:50,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/91e4f784d5db41e2b4e041b71f468e49 2024-11-12T18:14:50,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/152da823f119452095671601a4f1641a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/152da823f119452095671601a4f1641a 2024-11-12T18:14:50,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/cfa54f2c62d948c4a13d62fc0372b98f 2024-11-12T18:14:50,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/883f88407b25458c8f8c93eb965ce73e 2024-11-12T18:14:50,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bed694c2c5824fc7a87d3991e27322eb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/3d30b1497e8b404f95cfb93a82cc0823, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/73548bdae37e44638c8c500cf9d86a39, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/65cc9fb3c6574298a1c48ff2e55ec9a1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c203b1a07ff64e40a36ca80d0559d219, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f1ba52fac51d4b86bad7c1d2ee801379, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bbbfe68ae9bd4829a56df750decca2c4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/b7b0ebf7f5574351962262e189fdf20f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c63c3715dc7849b3962881138a854ea7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/fab9cb788c044110bf93b45c7756807d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/7b2a56884a684c1a93d6cd3f0f54787f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/aae86d7a11d64ef693c31354f09ccee4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/589a55b4aa444f7aa2277acf400ffaa0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/8270a83346094d67bd037f6398fc6517, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/1d992c7331bc436da14662d7e611d37b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e93cd96cdd5142d784a73e9992838dc9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/0a31b0672a864f1d9bcc6fff98e18578, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a9774d434a4a433d84aece773da8835f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/350c9a04176b441baeb3c8633f8b2266, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f2bd262d7c3e4089ab6f9817e93056db, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/d788e6e8c399444bb52b5d8799b07398, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/9f9bab04470d40558d3f4fb30ebfcccc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/64c0993e1f034791880794757a6e414d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a06781596960471f9cf26e02976e4dd6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/63b257b41eca449ebae1cab0a7988d67] to archive 2024-11-12T18:14:50,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:14:50,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bed694c2c5824fc7a87d3991e27322eb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bed694c2c5824fc7a87d3991e27322eb 2024-11-12T18:14:50,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/3d30b1497e8b404f95cfb93a82cc0823 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/3d30b1497e8b404f95cfb93a82cc0823 2024-11-12T18:14:50,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/73548bdae37e44638c8c500cf9d86a39 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/73548bdae37e44638c8c500cf9d86a39 2024-11-12T18:14:50,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/65cc9fb3c6574298a1c48ff2e55ec9a1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/65cc9fb3c6574298a1c48ff2e55ec9a1 2024-11-12T18:14:50,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c203b1a07ff64e40a36ca80d0559d219 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c203b1a07ff64e40a36ca80d0559d219 2024-11-12T18:14:50,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f1ba52fac51d4b86bad7c1d2ee801379 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f1ba52fac51d4b86bad7c1d2ee801379 2024-11-12T18:14:50,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bbbfe68ae9bd4829a56df750decca2c4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/bbbfe68ae9bd4829a56df750decca2c4 2024-11-12T18:14:50,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/b7b0ebf7f5574351962262e189fdf20f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/b7b0ebf7f5574351962262e189fdf20f 2024-11-12T18:14:50,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c63c3715dc7849b3962881138a854ea7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c63c3715dc7849b3962881138a854ea7 2024-11-12T18:14:50,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/fab9cb788c044110bf93b45c7756807d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/fab9cb788c044110bf93b45c7756807d 2024-11-12T18:14:50,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/7b2a56884a684c1a93d6cd3f0f54787f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/7b2a56884a684c1a93d6cd3f0f54787f 2024-11-12T18:14:50,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/aae86d7a11d64ef693c31354f09ccee4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/aae86d7a11d64ef693c31354f09ccee4 2024-11-12T18:14:50,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/589a55b4aa444f7aa2277acf400ffaa0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/589a55b4aa444f7aa2277acf400ffaa0 2024-11-12T18:14:50,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/8270a83346094d67bd037f6398fc6517 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/8270a83346094d67bd037f6398fc6517 2024-11-12T18:14:50,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/1d992c7331bc436da14662d7e611d37b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/1d992c7331bc436da14662d7e611d37b 2024-11-12T18:14:50,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e93cd96cdd5142d784a73e9992838dc9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e93cd96cdd5142d784a73e9992838dc9 2024-11-12T18:14:50,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/0a31b0672a864f1d9bcc6fff98e18578 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/0a31b0672a864f1d9bcc6fff98e18578 2024-11-12T18:14:50,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a9774d434a4a433d84aece773da8835f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a9774d434a4a433d84aece773da8835f 2024-11-12T18:14:50,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/350c9a04176b441baeb3c8633f8b2266 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/350c9a04176b441baeb3c8633f8b2266 2024-11-12T18:14:50,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f2bd262d7c3e4089ab6f9817e93056db to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/f2bd262d7c3e4089ab6f9817e93056db 2024-11-12T18:14:50,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/d788e6e8c399444bb52b5d8799b07398 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/d788e6e8c399444bb52b5d8799b07398 2024-11-12T18:14:50,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/9f9bab04470d40558d3f4fb30ebfcccc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/9f9bab04470d40558d3f4fb30ebfcccc 2024-11-12T18:14:50,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/64c0993e1f034791880794757a6e414d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/64c0993e1f034791880794757a6e414d 2024-11-12T18:14:50,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a06781596960471f9cf26e02976e4dd6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/a06781596960471f9cf26e02976e4dd6 2024-11-12T18:14:50,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/63b257b41eca449ebae1cab0a7988d67 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/63b257b41eca449ebae1cab0a7988d67 2024-11-12T18:14:50,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/618fd34f355d48db9ca1a7a6eeae1d96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1f4f830898384807a294ecaa67a848d4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/dc3abcfacbfd438b8c540735d15386de, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6637477017bf4760b37d48c20de42f9d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8ad0456cd20740a6919945e6dfb29f5f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c38aa06546e34735a12a7054cc1f1012, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c80e0e707e9d4ba0b0a3b5a7d1678494, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d4f0dc9e95ef43dfa28c6b332d14d5b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/be3d5189874a43aaafe4e0c779e44cc1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c481cb34c8de421cb4b3c831092c51b0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6b6f846dfe074a7aa79cb3b90da539e2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d7c218c5b15a40e08d2d597cd64a7565, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8b393569ff1c495e88ddd0734cd47f96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/4fba4df0cd93410c84f2f69d8cfdbf07, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/0af8bb0c22294b2e8e5b6371cc3e5879, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1b6021ee21774f02bb8349be0cdd8f9d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e0af43577b414009a4ab7c623b7fff74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8786a16d45f4453494f0de512ac7fe7e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/71c5d2be78494e8c902d66ab0b98916c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c9a3203fd904480ca42b2ab36b10cd8b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9af6c65f31c64195848e99397de953ae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/887d0cb2b8344fb1aeaac051521c9ea7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/af397aed58ee48b181680b0e3306588e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/523700452e3448329c1b373806646fa3] to archive 2024-11-12T18:14:50,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:14:50,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/618fd34f355d48db9ca1a7a6eeae1d96 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/618fd34f355d48db9ca1a7a6eeae1d96 2024-11-12T18:14:50,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1f4f830898384807a294ecaa67a848d4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1f4f830898384807a294ecaa67a848d4 2024-11-12T18:14:50,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/dc3abcfacbfd438b8c540735d15386de to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/dc3abcfacbfd438b8c540735d15386de 2024-11-12T18:14:50,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6637477017bf4760b37d48c20de42f9d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6637477017bf4760b37d48c20de42f9d 2024-11-12T18:14:50,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8ad0456cd20740a6919945e6dfb29f5f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8ad0456cd20740a6919945e6dfb29f5f 2024-11-12T18:14:50,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c38aa06546e34735a12a7054cc1f1012 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c38aa06546e34735a12a7054cc1f1012 2024-11-12T18:14:50,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c80e0e707e9d4ba0b0a3b5a7d1678494 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c80e0e707e9d4ba0b0a3b5a7d1678494 2024-11-12T18:14:50,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d4f0dc9e95ef43dfa28c6b332d14d5b1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d4f0dc9e95ef43dfa28c6b332d14d5b1 2024-11-12T18:14:50,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/be3d5189874a43aaafe4e0c779e44cc1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/be3d5189874a43aaafe4e0c779e44cc1 2024-11-12T18:14:50,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c481cb34c8de421cb4b3c831092c51b0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c481cb34c8de421cb4b3c831092c51b0 2024-11-12T18:14:50,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6b6f846dfe074a7aa79cb3b90da539e2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/6b6f846dfe074a7aa79cb3b90da539e2 2024-11-12T18:14:50,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d7c218c5b15a40e08d2d597cd64a7565 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/d7c218c5b15a40e08d2d597cd64a7565 2024-11-12T18:14:50,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8b393569ff1c495e88ddd0734cd47f96 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8b393569ff1c495e88ddd0734cd47f96 2024-11-12T18:14:50,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/4fba4df0cd93410c84f2f69d8cfdbf07 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/4fba4df0cd93410c84f2f69d8cfdbf07 2024-11-12T18:14:50,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/0af8bb0c22294b2e8e5b6371cc3e5879 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/0af8bb0c22294b2e8e5b6371cc3e5879 2024-11-12T18:14:50,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1b6021ee21774f02bb8349be0cdd8f9d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/1b6021ee21774f02bb8349be0cdd8f9d 2024-11-12T18:14:50,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e0af43577b414009a4ab7c623b7fff74 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e0af43577b414009a4ab7c623b7fff74 2024-11-12T18:14:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8786a16d45f4453494f0de512ac7fe7e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/8786a16d45f4453494f0de512ac7fe7e 2024-11-12T18:14:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/71c5d2be78494e8c902d66ab0b98916c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/71c5d2be78494e8c902d66ab0b98916c 2024-11-12T18:14:50,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c9a3203fd904480ca42b2ab36b10cd8b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c9a3203fd904480ca42b2ab36b10cd8b 2024-11-12T18:14:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9af6c65f31c64195848e99397de953ae to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9af6c65f31c64195848e99397de953ae 2024-11-12T18:14:50,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/887d0cb2b8344fb1aeaac051521c9ea7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/887d0cb2b8344fb1aeaac051521c9ea7 2024-11-12T18:14:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/af397aed58ee48b181680b0e3306588e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/af397aed58ee48b181680b0e3306588e 2024-11-12T18:14:50,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/523700452e3448329c1b373806646fa3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/523700452e3448329c1b373806646fa3 2024-11-12T18:14:50,295 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/recovered.edits/956.seqid, newMaxSeqId=956, maxSeqId=4 2024-11-12T18:14:50,296 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. 2024-11-12T18:14:50,296 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for ae79f5068ec1983ef2be0dd82f4139ba: Waiting for close lock at 1731435288821Running coprocessor pre-close hooks at 1731435288821Disabling compacts and flushes for region at 1731435288821Disabling writes for close at 1731435288821Obtaining lock to block concurrent updates at 1731435288821Preparing flush snapshotting stores in ae79f5068ec1983ef2be0dd82f4139ba at 1731435288821Finished memstore snapshotting TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731435288821Flushing stores of TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba. at 1731435288822 (+1 ms)Flushing ae79f5068ec1983ef2be0dd82f4139ba/A: creating writer at 1731435288822Flushing mob file ae79f5068ec1983ef2be0dd82f4139ba/A: appending metadata at 1731435288827 (+5 ms)Flushing mob file ae79f5068ec1983ef2be0dd82f4139ba/A: closing flushed file at 1731435288827Flushing ae79f5068ec1983ef2be0dd82f4139ba/A: appending metadata at 1731435289238 (+411 ms)Flushing ae79f5068ec1983ef2be0dd82f4139ba/A: closing flushed file at 1731435289238Flushing ae79f5068ec1983ef2be0dd82f4139ba/B: creating writer at 1731435289647 (+409 ms)Flushing ae79f5068ec1983ef2be0dd82f4139ba/B: appending metadata at 1731435289650 (+3 ms)Flushing ae79f5068ec1983ef2be0dd82f4139ba/B: closing flushed file at 1731435289650Flushing ae79f5068ec1983ef2be0dd82f4139ba/C: creating writer at 1731435290061 (+411 ms)Flushing ae79f5068ec1983ef2be0dd82f4139ba/C: appending metadata at 1731435290064 (+3 ms)Flushing ae79f5068ec1983ef2be0dd82f4139ba/C: closing flushed file at 1731435290064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d49333: reopening flushed file at 1731435290084 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f7e0ff5: reopening flushed file at 1731435290092 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c55fec5: reopening flushed file at 1731435290106 (+14 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae79f5068ec1983ef2be0dd82f4139ba in 1300ms, sequenceid=953, compaction requested=true at 1731435290121 (+15 ms)Writing region close event to WAL at 1731435290291 (+170 ms)Running coprocessor post-close hooks at 1731435290296 (+5 ms)Closed at 1731435290296 2024-11-12T18:14:50,298 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,299 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ae79f5068ec1983ef2be0dd82f4139ba, regionState=CLOSED 2024-11-12T18:14:50,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:14:50,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T18:14:50,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure ae79f5068ec1983ef2be0dd82f4139ba, server=9911683f163c,43757,1731435233300 in 1.6340 sec 2024-11-12T18:14:50,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-12T18:14:50,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae79f5068ec1983ef2be0dd82f4139ba, UNASSIGN in 1.6410 sec 2024-11-12T18:14:50,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T18:14:50,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6460 sec 2024-11-12T18:14:50,311 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435290311"}]},"ts":"1731435290311"} 2024-11-12T18:14:50,313 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T18:14:50,313 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T18:14:50,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6590 sec 2024-11-12T18:14:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T18:14:50,801 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:50,801 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T18:14:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:50,803 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T18:14:50,805 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:50,809 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,812 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/recovered.edits] 2024-11-12T18:14:50,816 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/0199ea092c5f40259cf0440f98777a90 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/0199ea092c5f40259cf0440f98777a90 2024-11-12T18:14:50,817 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/282a9515d70447c0b34e83ffb8882905 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/282a9515d70447c0b34e83ffb8882905 2024-11-12T18:14:50,818 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3bb28d2aed914e81b29d2f7c1d0c0aa9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3bb28d2aed914e81b29d2f7c1d0c0aa9 2024-11-12T18:14:50,821 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3d7cf014f25043ef8d2a788faad981f4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/A/3d7cf014f25043ef8d2a788faad981f4 2024-11-12T18:14:50,824 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/2e0758e74a02419dbde28077264a8bd5 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/2e0758e74a02419dbde28077264a8bd5 2024-11-12T18:14:50,826 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/4c1467b6315840aa902a1a5bdd6943be to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/4c1467b6315840aa902a1a5bdd6943be 2024-11-12T18:14:50,828 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c1e609cebf2a4e90a5585b034d3942b5 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/c1e609cebf2a4e90a5585b034d3942b5 2024-11-12T18:14:50,830 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e9c2a94f13234b3e827f742d613af403 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/B/e9c2a94f13234b3e827f742d613af403 2024-11-12T18:14:50,833 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/60f7cb53c20547de87326750c1d1e502 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/60f7cb53c20547de87326750c1d1e502 2024-11-12T18:14:50,835 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9cfa8e56a8df4e3eadcf1257c01f3cbf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/9cfa8e56a8df4e3eadcf1257c01f3cbf 2024-11-12T18:14:50,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c4a270dc5a224a249a6ec081dc3974c4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/c4a270dc5a224a249a6ec081dc3974c4 2024-11-12T18:14:50,837 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e9ff3c94830c4684be7e57e8f24d53c8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/C/e9ff3c94830c4684be7e57e8f24d53c8 2024-11-12T18:14:50,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/recovered.edits/956.seqid to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba/recovered.edits/956.seqid 2024-11-12T18:14:50,844 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,844 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T18:14:50,844 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T18:14:50,846 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T18:14:50,855 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111206522552e7314d7f94c7459cbf1e70d3_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111206522552e7314d7f94c7459cbf1e70d3_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112069640ae20c245f18992eddc49f8fbea_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112069640ae20c245f18992eddc49f8fbea_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111221dab2f6555a49efa781cb338950b8a2_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111221dab2f6555a49efa781cb338950b8a2_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,861 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112284e78d0f81b4a70a8fa89df6d856ad7_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112284e78d0f81b4a70a8fa89df6d856ad7_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111231a687282cbe4b7c9212272e523921d1_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111231a687282cbe4b7c9212272e523921d1_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,865 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112394aa2f714c9410ab930a21509e012f9_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112394aa2f714c9410ab930a21509e012f9_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123a76773f4cff49a197f64fa71437b7bd_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123a76773f4cff49a197f64fa71437b7bd_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260518e1974384a5c9b94b40ab512f1a5_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260518e1974384a5c9b94b40ab512f1a5_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111265e85d6d4aab42b0947aac3e9b537d8d_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111265e85d6d4aab42b0947aac3e9b537d8d_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126a8b88fc56d24e11908ce5f4f7ad33b3_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126a8b88fc56d24e11908ce5f4f7ad33b3_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,877 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127978dd52ac11478e854fcd29a5a3eb31_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127978dd52ac11478e854fcd29a5a3eb31_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e5a2b07f8944b0b978c961fb8ae44ac_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e5a2b07f8944b0b978c961fb8ae44ac_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,882 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e6998b1c595412d82799082424fc809_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129e6998b1c595412d82799082424fc809_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,885 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ac1f60b576ef41dfbe13e68b4ca1150a_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ac1f60b576ef41dfbe13e68b4ca1150a_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,887 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ae2f122d5794449d8ca44d21a0fec035_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ae2f122d5794449d8ca44d21a0fec035_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,890 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bf700979a75c45adadc0f6c6683051f5_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bf700979a75c45adadc0f6c6683051f5_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,893 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112df688f168c1048c68bb692282c0dff51_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112df688f168c1048c68bb692282c0dff51_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,895 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e91710327ada4fc8ac2f509387aeca20_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e91710327ada4fc8ac2f509387aeca20_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,897 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ecdc38cb961949d399c272b6bc02ebaa_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ecdc38cb961949d399c272b6bc02ebaa_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,899 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ed5c3265cf604512ad360c304701af56_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ed5c3265cf604512ad360c304701af56_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,901 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f43079cb8f234b6ab92858f3c357ca25_ae79f5068ec1983ef2be0dd82f4139ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f43079cb8f234b6ab92858f3c357ca25_ae79f5068ec1983ef2be0dd82f4139ba 2024-11-12T18:14:50,902 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T18:14:50,904 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:50,907 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T18:14:50,910 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T18:14:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T18:14:50,911 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:50,911 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T18:14:50,912 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731435290911"}]},"ts":"9223372036854775807"} 2024-11-12T18:14:50,915 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T18:14:50,915 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae79f5068ec1983ef2be0dd82f4139ba, NAME => 'TestAcidGuarantees,,1731435260688.ae79f5068ec1983ef2be0dd82f4139ba.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T18:14:50,915 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T18:14:50,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731435290915"}]},"ts":"9223372036854775807"} 2024-11-12T18:14:50,917 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T18:14:50,919 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:50,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 118 msec 2024-11-12T18:14:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T18:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T18:14:51,121 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:51,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=759 (was 858), ProcessCount=11 (was 11), AvailableMemoryMB=2059 (was 2980) 2024-11-12T18:14:51,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=759, ProcessCount=11, AvailableMemoryMB=2059 2024-11-12T18:14:51,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:14:51,151 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T18:14:51,154 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:14:51,154 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:51,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-12T18:14:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T18:14:51,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:14:51,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742063_1239 (size=1061) 2024-11-12T18:14:51,172 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:14:51,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742064_1240 (size=53) 2024-11-12T18:14:51,242 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T18:14:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T18:14:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T18:14:51,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:51,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a393ec6eeb0c6b3b0d3ff894a1f1a04b, disabling compactions & flushes 2024-11-12T18:14:51,581 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. after waiting 0 ms 2024-11-12T18:14:51,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,581 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: Waiting for close lock at 1731435291581Disabling compacts and flushes for region at 1731435291581Disabling writes for close at 1731435291581Writing region close event to WAL at 1731435291581Closed at 1731435291581 2024-11-12T18:14:51,583 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:14:51,583 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731435291583"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435291583"}]},"ts":"1731435291583"} 2024-11-12T18:14:51,585 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T18:14:51,585 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:14:51,585 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435291585"}]},"ts":"1731435291585"} 2024-11-12T18:14:51,587 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T18:14:51,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, ASSIGN}] 2024-11-12T18:14:51,588 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, ASSIGN 2024-11-12T18:14:51,589 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, ASSIGN; state=OFFLINE, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=false 2024-11-12T18:14:51,739 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a393ec6eeb0c6b3b0d3ff894a1f1a04b, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:51,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, ASSIGN because future has completed 2024-11-12T18:14:51,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T18:14:51,896 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,896 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:14:51,897 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,897 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:14:51,897 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,897 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,898 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,899 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:51,899 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a393ec6eeb0c6b3b0d3ff894a1f1a04b columnFamilyName A 2024-11-12T18:14:51,899 DEBUG [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:51,900 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.HStore(327): Store=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:51,900 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,901 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:51,901 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a393ec6eeb0c6b3b0d3ff894a1f1a04b columnFamilyName B 2024-11-12T18:14:51,901 DEBUG [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:51,901 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.HStore(327): Store=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:51,902 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,902 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:14:51,903 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a393ec6eeb0c6b3b0d3ff894a1f1a04b columnFamilyName C 2024-11-12T18:14:51,903 DEBUG [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:14:51,903 INFO [StoreOpener-a393ec6eeb0c6b3b0d3ff894a1f1a04b-1 {}] regionserver.HStore(327): Store=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:14:51,903 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,903 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,904 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,904 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,905 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,905 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,906 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:14:51,907 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,909 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:14:51,909 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened a393ec6eeb0c6b3b0d3ff894a1f1a04b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72783391, jitterRate=0.08455704152584076}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:14:51,909 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:51,910 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: Running coprocessor pre-open hook at 1731435291897Writing region info on filesystem at 1731435291897Initializing all the Stores at 1731435291897Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435291898 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435291898Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435291898Cleaning up temporary data from old regions at 1731435291905 (+7 ms)Running coprocessor post-open hooks at 1731435291909 (+4 ms)Region opened successfully at 1731435291910 (+1 ms) 2024-11-12T18:14:51,911 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., pid=63, masterSystemTime=1731435291893 2024-11-12T18:14:51,912 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,912 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:51,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a393ec6eeb0c6b3b0d3ff894a1f1a04b, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:14:51,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:14:51,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T18:14:51,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 in 174 msec 2024-11-12T18:14:51,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-12T18:14:51,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, ASSIGN in 329 msec 2024-11-12T18:14:51,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:14:51,919 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435291919"}]},"ts":"1731435291919"} 2024-11-12T18:14:51,920 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T18:14:51,921 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:14:51,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 769 msec 2024-11-12T18:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-12T18:14:52,291 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:52,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,293 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7293679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,295 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5467ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,296 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d9b3121 2024-11-12T18:14:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:14:52,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:14:52,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b095438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,306 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ef7e49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,308 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161ea75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794a417d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,311 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3beb371d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59057ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3ad4a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c057a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bca9de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efe6821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797ddc89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399cd69e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6310da8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52693ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,327 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f214bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1687c764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,331 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2285e5df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,333 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335eb394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:14:52,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:14:52,338 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:14:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:14:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:14:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2961ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:14:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:14:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:14:52,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:14:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776f8a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:14:52,340 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,341 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,341 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:52,346 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T18:14:52,349 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:52,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:52,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:52,350 DEBUG [Thread-1134 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,350 DEBUG [Thread-1134 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T18:14:52,354 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:52,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:52,365 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,375 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,383 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,387 DEBUG [Thread-1132 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,387 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,407 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:14:52,408 DEBUG [Thread-1123 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:14:52,409 DEBUG [Thread-1123 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:14:52,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:14:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:14:52,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8d6d750e96924bf29e1a116c85d805b1 is 50, key is test_row_0/A:col10/1731435292354/Put/seqid=0 2024-11-12T18:14:52,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435352416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435352418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435292405, max timestamp=1731435292432Num uniques 859; 2024-11-12T18:14:52,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435292405, max timestamp=1731435292432Num uniques 859; 2024-11-12T18:14:52,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435292405, max timestamp=1731435292432Num uniques 859; 2024-11-12T18:14:52,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:52,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T18:14:52,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742065_1241 (size=12001) 2024-11-12T18:14:52,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8d6d750e96924bf29e1a116c85d805b1 2024-11-12T18:14:52,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T18:14:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:52,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:52,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:52,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:52,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:52,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/0023264875654149ba7157a706a4346b is 50, key is test_row_0/B:col10/1731435292354/Put/seqid=0 2024-11-12T18:14:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435352504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435352505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435352509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435352551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435352551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742066_1242 (size=12001) 2024-11-12T18:14:52,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/0023264875654149ba7157a706a4346b 2024-11-12T18:14:52,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/4ff2be856a6a4f9c84b79089f5256de1 is 50, key is test_row_0/C:col10/1731435292354/Put/seqid=0 2024-11-12T18:14:52,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742067_1243 (size=12001) 2024-11-12T18:14:52,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/4ff2be856a6a4f9c84b79089f5256de1 2024-11-12T18:14:52,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435352621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435352621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435352622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8d6d750e96924bf29e1a116c85d805b1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8d6d750e96924bf29e1a116c85d805b1 2024-11-12T18:14:52,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8d6d750e96924bf29e1a116c85d805b1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T18:14:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/0023264875654149ba7157a706a4346b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/0023264875654149ba7157a706a4346b 2024-11-12T18:14:52,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/0023264875654149ba7157a706a4346b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T18:14:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/4ff2be856a6a4f9c84b79089f5256de1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/4ff2be856a6a4f9c84b79089f5256de1 2024-11-12T18:14:52,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/4ff2be856a6a4f9c84b79089f5256de1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T18:14:52,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 291ms, sequenceid=13, compaction requested=false 2024-11-12T18:14:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T18:14:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:52,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T18:14:52,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:52,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-12T18:14:52,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:52,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:52,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:52,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:52,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:52,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/56932395b2cb43e0b1f8a31796074496 is 50, key is test_row_0/A:col10/1731435292482/Put/seqid=0 2024-11-12T18:14:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T18:14:52,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742068_1244 (size=12001) 2024-11-12T18:14:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:52,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435352805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435352806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435352831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435352831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435352831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435352921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435352920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T18:14:53,117 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/56932395b2cb43e0b1f8a31796074496 2024-11-12T18:14:53,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435353141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435353141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435353141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d96c5421238748289d8bc24656fece07 is 50, key is test_row_0/B:col10/1731435292482/Put/seqid=0 2024-11-12T18:14:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435353150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435353150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742069_1245 (size=12001) 2024-11-12T18:14:53,207 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d96c5421238748289d8bc24656fece07 2024-11-12T18:14:53,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ecf0bd2ec7f7426dbff92007566968e7 is 50, key is test_row_0/C:col10/1731435292482/Put/seqid=0 2024-11-12T18:14:53,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742070_1246 (size=12001) 2024-11-12T18:14:53,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435353461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435353461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T18:14:53,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ecf0bd2ec7f7426dbff92007566968e7 2024-11-12T18:14:53,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/56932395b2cb43e0b1f8a31796074496 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/56932395b2cb43e0b1f8a31796074496 2024-11-12T18:14:53,647 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/56932395b2cb43e0b1f8a31796074496, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T18:14:53,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d96c5421238748289d8bc24656fece07 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d96c5421238748289d8bc24656fece07 2024-11-12T18:14:53,654 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d96c5421238748289d8bc24656fece07, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T18:14:53,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ecf0bd2ec7f7426dbff92007566968e7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ecf0bd2ec7f7426dbff92007566968e7 2024-11-12T18:14:53,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ecf0bd2ec7f7426dbff92007566968e7, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T18:14:53,661 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=67.09 KB/68700 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1004ms, sequenceid=46, compaction requested=false 2024-11-12T18:14:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T18:14:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T18:14:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:53,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:14:53,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:53,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:53,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:53,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:53,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:53,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:53,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T18:14:53,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3140 sec 2024-11-12T18:14:53,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/eb9b37c337e54141b7d6cd17d167d248 is 50, key is test_row_0/A:col10/1731435292794/Put/seqid=0 2024-11-12T18:14:53,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3360 sec 2024-11-12T18:14:53,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742071_1247 (size=12001) 2024-11-12T18:14:53,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/eb9b37c337e54141b7d6cd17d167d248 2024-11-12T18:14:53,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/af40ca50a65d42c78546bac3e07a1792 is 50, key is test_row_0/B:col10/1731435292794/Put/seqid=0 2024-11-12T18:14:53,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742072_1248 (size=12001) 2024-11-12T18:14:53,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/af40ca50a65d42c78546bac3e07a1792 2024-11-12T18:14:53,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435353726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435353728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435353731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/a41af412fdfb46e2bdf37e6c4110a3b0 is 50, key is test_row_0/C:col10/1731435292794/Put/seqid=0 2024-11-12T18:14:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742073_1249 (size=12001) 2024-11-12T18:14:53,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/a41af412fdfb46e2bdf37e6c4110a3b0 2024-11-12T18:14:53,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/eb9b37c337e54141b7d6cd17d167d248 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb9b37c337e54141b7d6cd17d167d248 2024-11-12T18:14:53,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb9b37c337e54141b7d6cd17d167d248, entries=150, sequenceid=60, filesize=11.7 K 2024-11-12T18:14:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/af40ca50a65d42c78546bac3e07a1792 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/af40ca50a65d42c78546bac3e07a1792 2024-11-12T18:14:53,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/af40ca50a65d42c78546bac3e07a1792, entries=150, sequenceid=60, filesize=11.7 K 2024-11-12T18:14:53,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/a41af412fdfb46e2bdf37e6c4110a3b0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/a41af412fdfb46e2bdf37e6c4110a3b0 2024-11-12T18:14:53,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/a41af412fdfb46e2bdf37e6c4110a3b0, entries=150, sequenceid=60, filesize=11.7 K 2024-11-12T18:14:53,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 151ms, sequenceid=60, compaction requested=true 2024-11-12T18:14:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:53,816 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:53,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:53,817 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:53,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:53,818 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:53,818 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:53,818 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:14:53,818 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:53,818 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:14:53,818 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8d6d750e96924bf29e1a116c85d805b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/56932395b2cb43e0b1f8a31796074496, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb9b37c337e54141b7d6cd17d167d248] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=35.2 K 2024-11-12T18:14:53,818 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:53,818 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/0023264875654149ba7157a706a4346b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d96c5421238748289d8bc24656fece07, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/af40ca50a65d42c78546bac3e07a1792] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=35.2 K 2024-11-12T18:14:53,818 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d6d750e96924bf29e1a116c85d805b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731435292354 2024-11-12T18:14:53,818 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0023264875654149ba7157a706a4346b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731435292354 2024-11-12T18:14:53,819 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56932395b2cb43e0b1f8a31796074496, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731435292482 2024-11-12T18:14:53,819 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d96c5421238748289d8bc24656fece07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731435292482 2024-11-12T18:14:53,819 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting af40ca50a65d42c78546bac3e07a1792, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731435292792 2024-11-12T18:14:53,819 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb9b37c337e54141b7d6cd17d167d248, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731435292792 2024-11-12T18:14:53,829 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:53,829 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/9565c083e9504fe7bcdfe18c9be9eaee is 50, key is test_row_0/A:col10/1731435292794/Put/seqid=0 2024-11-12T18:14:53,836 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:53,836 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/bf1675c129fa4623bed8f6bad42a694a is 50, key is test_row_0/B:col10/1731435292794/Put/seqid=0 2024-11-12T18:14:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:53,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:14:53,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:53,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:53,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:53,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:53,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:53,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:53,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/d741b48d91de49278e58ef7858fc5372 is 50, key is test_row_0/A:col10/1731435293715/Put/seqid=0 2024-11-12T18:14:53,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742075_1251 (size=12104) 2024-11-12T18:14:53,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742074_1250 (size=12104) 2024-11-12T18:14:53,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435353866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435353866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435353867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742076_1252 (size=12001) 2024-11-12T18:14:53,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/d741b48d91de49278e58ef7858fc5372 2024-11-12T18:14:53,884 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/bf1675c129fa4623bed8f6bad42a694a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bf1675c129fa4623bed8f6bad42a694a 2024-11-12T18:14:53,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d5a5d5354b714c1e8f39d8e27d62f7a4 is 50, key is test_row_0/B:col10/1731435293715/Put/seqid=0 2024-11-12T18:14:53,898 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into bf1675c129fa4623bed8f6bad42a694a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:53,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:53,898 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435293816; duration=0sec 2024-11-12T18:14:53,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:53,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:14:53,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:53,904 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:53,904 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:14:53,904 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:53,904 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/4ff2be856a6a4f9c84b79089f5256de1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ecf0bd2ec7f7426dbff92007566968e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/a41af412fdfb46e2bdf37e6c4110a3b0] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=35.2 K 2024-11-12T18:14:53,904 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff2be856a6a4f9c84b79089f5256de1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731435292354 2024-11-12T18:14:53,905 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf0bd2ec7f7426dbff92007566968e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731435292482 2024-11-12T18:14:53,905 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a41af412fdfb46e2bdf37e6c4110a3b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731435292792 2024-11-12T18:14:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742077_1253 (size=12001) 2024-11-12T18:14:53,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d5a5d5354b714c1e8f39d8e27d62f7a4 2024-11-12T18:14:53,943 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:53,944 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/249de40a941b41238040c1fcdba33972 is 50, key is test_row_0/C:col10/1731435292794/Put/seqid=0 2024-11-12T18:14:53,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ba35be7bbb9643d7bf5cc9247ea1cb8c is 50, key is test_row_0/C:col10/1731435293715/Put/seqid=0 2024-11-12T18:14:53,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435353980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435353980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435353981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435353981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435353981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:53,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742078_1254 (size=12104) 2024-11-12T18:14:53,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742079_1255 (size=12001) 2024-11-12T18:14:53,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ba35be7bbb9643d7bf5cc9247ea1cb8c 2024-11-12T18:14:54,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/d741b48d91de49278e58ef7858fc5372 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/d741b48d91de49278e58ef7858fc5372 2024-11-12T18:14:54,008 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/249de40a941b41238040c1fcdba33972 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/249de40a941b41238040c1fcdba33972 2024-11-12T18:14:54,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/d741b48d91de49278e58ef7858fc5372, entries=150, sequenceid=81, filesize=11.7 K 2024-11-12T18:14:54,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d5a5d5354b714c1e8f39d8e27d62f7a4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d5a5d5354b714c1e8f39d8e27d62f7a4 2024-11-12T18:14:54,014 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 249de40a941b41238040c1fcdba33972(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:54,014 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,014 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435293817; duration=0sec 2024-11-12T18:14:54,015 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:54,015 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:14:54,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d5a5d5354b714c1e8f39d8e27d62f7a4, entries=150, sequenceid=81, filesize=11.7 K 2024-11-12T18:14:54,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ba35be7bbb9643d7bf5cc9247ea1cb8c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ba35be7bbb9643d7bf5cc9247ea1cb8c 2024-11-12T18:14:54,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ba35be7bbb9643d7bf5cc9247ea1cb8c, entries=150, sequenceid=81, filesize=11.7 K 2024-11-12T18:14:54,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 194ms, sequenceid=81, compaction requested=false 2024-11-12T18:14:54,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:54,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:14:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/674706f137f84165859f7c31940af430 is 50, key is test_row_0/A:col10/1731435293864/Put/seqid=0 2024-11-12T18:14:54,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742080_1256 (size=11997) 2024-11-12T18:14:54,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/674706f137f84165859f7c31940af430 2024-11-12T18:14:54,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/4137c8ef34074cfeb26feda4200823c6 is 50, key is test_row_0/B:col10/1731435293864/Put/seqid=0 2024-11-12T18:14:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435354239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435354240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435354247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742081_1257 (size=9657) 2024-11-12T18:14:54,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/4137c8ef34074cfeb26feda4200823c6 2024-11-12T18:14:54,286 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/9565c083e9504fe7bcdfe18c9be9eaee as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9565c083e9504fe7bcdfe18c9be9eaee 2024-11-12T18:14:54,295 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 9565c083e9504fe7bcdfe18c9be9eaee(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:54,295 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,295 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435293816; duration=0sec 2024-11-12T18:14:54,295 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:54,295 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:14:54,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/94c4ab84489e4f538916df0a958aaf6c is 50, key is test_row_0/C:col10/1731435293864/Put/seqid=0 2024-11-12T18:14:54,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742082_1258 (size=9657) 2024-11-12T18:14:54,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/94c4ab84489e4f538916df0a958aaf6c 2024-11-12T18:14:54,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/674706f137f84165859f7c31940af430 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/674706f137f84165859f7c31940af430 2024-11-12T18:14:54,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/674706f137f84165859f7c31940af430, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T18:14:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/4137c8ef34074cfeb26feda4200823c6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4137c8ef34074cfeb26feda4200823c6 2024-11-12T18:14:54,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435354351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435354351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4137c8ef34074cfeb26feda4200823c6, entries=100, sequenceid=96, filesize=9.4 K 2024-11-12T18:14:54,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/94c4ab84489e4f538916df0a958aaf6c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/94c4ab84489e4f538916df0a958aaf6c 2024-11-12T18:14:54,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/94c4ab84489e4f538916df0a958aaf6c, entries=100, sequenceid=96, filesize=9.4 K 2024-11-12T18:14:54,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 168ms, sequenceid=96, compaction requested=true 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:54,365 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:14:54,365 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:54,367 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:54,367 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:14:54,367 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,367 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bf1675c129fa4623bed8f6bad42a694a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d5a5d5354b714c1e8f39d8e27d62f7a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4137c8ef34074cfeb26feda4200823c6] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=33.0 K 2024-11-12T18:14:54,367 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:54,367 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:14:54,367 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,367 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9565c083e9504fe7bcdfe18c9be9eaee, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/d741b48d91de49278e58ef7858fc5372, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/674706f137f84165859f7c31940af430] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=35.3 K 2024-11-12T18:14:54,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:14:54,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:54,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:54,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:54,368 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1675c129fa4623bed8f6bad42a694a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731435292792 2024-11-12T18:14:54,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,368 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9565c083e9504fe7bcdfe18c9be9eaee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731435292792 2024-11-12T18:14:54,370 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a5d5354b714c1e8f39d8e27d62f7a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731435293715 2024-11-12T18:14:54,371 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d741b48d91de49278e58ef7858fc5372, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731435293715 2024-11-12T18:14:54,371 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4137c8ef34074cfeb26feda4200823c6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435293864 2024-11-12T18:14:54,372 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674706f137f84165859f7c31940af430, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435293854 2024-11-12T18:14:54,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/b245b8c912664efea93e0ffb2d9f1e41 is 50, key is test_row_0/A:col10/1731435294364/Put/seqid=0 2024-11-12T18:14:54,407 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:54,408 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/03bb15c0dadc43adbff5e58f889b3459 is 50, key is test_row_0/A:col10/1731435293864/Put/seqid=0 2024-11-12T18:14:54,423 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:54,423 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d6164778c0994101afaa000503d5ec46 is 50, key is test_row_0/B:col10/1731435293864/Put/seqid=0 2024-11-12T18:14:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742083_1259 (size=12001) 2024-11-12T18:14:54,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435354438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742084_1260 (size=12207) 2024-11-12T18:14:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742085_1261 (size=12207) 2024-11-12T18:14:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T18:14:54,502 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:54,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T18:14:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T18:14:54,505 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:54,506 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:54,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:54,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435354550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435354560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435354561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T18:14:54,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T18:14:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:54,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435354761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T18:14:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T18:14:54,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/b245b8c912664efea93e0ffb2d9f1e41 2024-11-12T18:14:54,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/7a4a15375e25484391f1df298e96eddf is 50, key is test_row_0/B:col10/1731435294364/Put/seqid=0 2024-11-12T18:14:54,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742086_1262 (size=12001) 2024-11-12T18:14:54,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/7a4a15375e25484391f1df298e96eddf 2024-11-12T18:14:54,861 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/03bb15c0dadc43adbff5e58f889b3459 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/03bb15c0dadc43adbff5e58f889b3459 2024-11-12T18:14:54,867 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 03bb15c0dadc43adbff5e58f889b3459(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:54,867 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,867 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435294365; duration=0sec 2024-11-12T18:14:54,868 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:54,868 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:14:54,868 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:54,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/23ff15ff18fb40ea8037bcddf524faa4 is 50, key is test_row_0/C:col10/1731435294364/Put/seqid=0 2024-11-12T18:14:54,870 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:54,870 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:14:54,870 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,871 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/249de40a941b41238040c1fcdba33972, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ba35be7bbb9643d7bf5cc9247ea1cb8c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/94c4ab84489e4f538916df0a958aaf6c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=33.0 K 2024-11-12T18:14:54,871 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249de40a941b41238040c1fcdba33972, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731435292792 2024-11-12T18:14:54,872 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba35be7bbb9643d7bf5cc9247ea1cb8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731435293715 2024-11-12T18:14:54,872 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c4ab84489e4f538916df0a958aaf6c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435293864 2024-11-12T18:14:54,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435354871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435354871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:54,878 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/d6164778c0994101afaa000503d5ec46 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d6164778c0994101afaa000503d5ec46 2024-11-12T18:14:54,882 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:54,883 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5881c1b14d54400e9fe3afa0541377dc is 50, key is test_row_0/C:col10/1731435293864/Put/seqid=0 2024-11-12T18:14:54,885 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into d6164778c0994101afaa000503d5ec46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:54,885 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,885 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435294365; duration=0sec 2024-11-12T18:14:54,885 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:54,885 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:14:54,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742087_1263 (size=12001) 2024-11-12T18:14:54,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/23ff15ff18fb40ea8037bcddf524faa4 2024-11-12T18:14:54,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/b245b8c912664efea93e0ffb2d9f1e41 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/b245b8c912664efea93e0ffb2d9f1e41 2024-11-12T18:14:54,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/b245b8c912664efea93e0ffb2d9f1e41, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T18:14:54,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/7a4a15375e25484391f1df298e96eddf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7a4a15375e25484391f1df298e96eddf 2024-11-12T18:14:54,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7a4a15375e25484391f1df298e96eddf, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T18:14:54,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/23ff15ff18fb40ea8037bcddf524faa4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/23ff15ff18fb40ea8037bcddf524faa4 2024-11-12T18:14:54,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742088_1264 (size=12207) 2024-11-12T18:14:54,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/23ff15ff18fb40ea8037bcddf524faa4, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T18:14:54,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 556ms, sequenceid=117, compaction requested=false 2024-11-12T18:14:54,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,931 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5881c1b14d54400e9fe3afa0541377dc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5881c1b14d54400e9fe3afa0541377dc 2024-11-12T18:14:54,948 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 5881c1b14d54400e9fe3afa0541377dc(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:54,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:54,949 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435294365; duration=0sec 2024-11-12T18:14:54,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:54,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:14:54,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:54,968 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8eb1fde002054ee58b833e2c5b96030f is 50, key is test_row_0/A:col10/1731435294435/Put/seqid=0 2024-11-12T18:14:54,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742089_1265 (size=12001) 2024-11-12T18:14:54,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8eb1fde002054ee58b833e2c5b96030f 2024-11-12T18:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:54,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:55,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9ca1c777527f48988dacfd060cb7c4eb is 50, key is test_row_0/B:col10/1731435294435/Put/seqid=0 2024-11-12T18:14:55,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742090_1266 (size=12001) 2024-11-12T18:14:55,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435355056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435355058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435355070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T18:14:55,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435355170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435355170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435355380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435355381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435355381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435355390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,451 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9ca1c777527f48988dacfd060cb7c4eb 2024-11-12T18:14:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/7618138ed0e64b579f7ed0d097944ced is 50, key is test_row_0/C:col10/1731435294435/Put/seqid=0 2024-11-12T18:14:55,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742091_1267 (size=12001) 2024-11-12T18:14:55,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/7618138ed0e64b579f7ed0d097944ced 2024-11-12T18:14:55,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8eb1fde002054ee58b833e2c5b96030f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8eb1fde002054ee58b833e2c5b96030f 2024-11-12T18:14:55,493 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8eb1fde002054ee58b833e2c5b96030f, entries=150, sequenceid=132, filesize=11.7 K 2024-11-12T18:14:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9ca1c777527f48988dacfd060cb7c4eb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9ca1c777527f48988dacfd060cb7c4eb 2024-11-12T18:14:55,499 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9ca1c777527f48988dacfd060cb7c4eb, entries=150, sequenceid=132, filesize=11.7 K 2024-11-12T18:14:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/7618138ed0e64b579f7ed0d097944ced as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7618138ed0e64b579f7ed0d097944ced 2024-11-12T18:14:55,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7618138ed0e64b579f7ed0d097944ced, entries=150, sequenceid=132, filesize=11.7 K 2024-11-12T18:14:55,507 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 538ms, sequenceid=132, compaction requested=true 2024-11-12T18:14:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T18:14:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T18:14:55,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T18:14:55,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0020 sec 2024-11-12T18:14:55,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0090 sec 2024-11-12T18:14:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:55,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T18:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:55,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8797d2458094405f8a4cf705c6de5423 is 50, key is test_row_0/A:col10/1731435295597/Put/seqid=0 2024-11-12T18:14:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T18:14:55,641 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:55,643 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-12T18:14:55,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T18:14:55,647 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:55,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:55,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742092_1268 (size=14541) 2024-11-12T18:14:55,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8797d2458094405f8a4cf705c6de5423 2024-11-12T18:14:55,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435355659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/21bb1f97bf2a4d1cbc1668282f049faa is 50, key is test_row_0/B:col10/1731435295597/Put/seqid=0 2024-11-12T18:14:55,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742093_1269 (size=12151) 2024-11-12T18:14:55,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/21bb1f97bf2a4d1cbc1668282f049faa 2024-11-12T18:14:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435355691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435355691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/7bc757dd045549c49b5a31a2b1ce3ee5 is 50, key is test_row_0/C:col10/1731435295597/Put/seqid=0 2024-11-12T18:14:55,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742094_1270 (size=12151) 2024-11-12T18:14:55,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/7bc757dd045549c49b5a31a2b1ce3ee5 2024-11-12T18:14:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T18:14:55,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8797d2458094405f8a4cf705c6de5423 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8797d2458094405f8a4cf705c6de5423 2024-11-12T18:14:55,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8797d2458094405f8a4cf705c6de5423, entries=200, sequenceid=153, filesize=14.2 K 2024-11-12T18:14:55,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/21bb1f97bf2a4d1cbc1668282f049faa as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/21bb1f97bf2a4d1cbc1668282f049faa 2024-11-12T18:14:55,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/21bb1f97bf2a4d1cbc1668282f049faa, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T18:14:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/7bc757dd045549c49b5a31a2b1ce3ee5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7bc757dd045549c49b5a31a2b1ce3ee5 2024-11-12T18:14:55,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435355770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:55,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7bc757dd045549c49b5a31a2b1ce3ee5, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T18:14:55,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 191ms, sequenceid=153, compaction requested=true 2024-11-12T18:14:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:55,790 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:55,790 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:14:55,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:55,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:55,792 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:55,792 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:14:55,792 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:55,792 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/03bb15c0dadc43adbff5e58f889b3459, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/b245b8c912664efea93e0ffb2d9f1e41, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8eb1fde002054ee58b833e2c5b96030f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8797d2458094405f8a4cf705c6de5423] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=49.6 K 2024-11-12T18:14:55,792 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:14:55,793 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:14:55,793 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:55,793 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d6164778c0994101afaa000503d5ec46, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7a4a15375e25484391f1df298e96eddf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9ca1c777527f48988dacfd060cb7c4eb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/21bb1f97bf2a4d1cbc1668282f049faa] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=47.2 K 2024-11-12T18:14:55,793 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03bb15c0dadc43adbff5e58f889b3459, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435293843 2024-11-12T18:14:55,793 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d6164778c0994101afaa000503d5ec46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435293843 2024-11-12T18:14:55,794 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting b245b8c912664efea93e0ffb2d9f1e41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731435294237 2024-11-12T18:14:55,794 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb1fde002054ee58b833e2c5b96030f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731435294384 2024-11-12T18:14:55,794 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8797d2458094405f8a4cf705c6de5423, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435295036 2024-11-12T18:14:55,794 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a4a15375e25484391f1df298e96eddf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731435294237 2024-11-12T18:14:55,795 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca1c777527f48988dacfd060cb7c4eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731435294384 2024-11-12T18:14:55,795 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bb1f97bf2a4d1cbc1668282f049faa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435295036 2024-11-12T18:14:55,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T18:14:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:55,802 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:14:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:55,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:55,814 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#224 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:55,815 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/ea8f608b6003449eb7d4f7d16ba4ffd6 is 50, key is test_row_0/A:col10/1731435295597/Put/seqid=0 2024-11-12T18:14:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3cc7b95394be4b1594453cc3296a7ba8 is 50, key is test_row_0/A:col10/1731435295651/Put/seqid=0 2024-11-12T18:14:55,822 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:14:55,831 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T18:14:55,832 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/7c6b955bdd9d4286a1dbc57b193bdc0e is 50, key is test_row_0/B:col10/1731435295597/Put/seqid=0 2024-11-12T18:14:55,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742096_1272 (size=12151) 2024-11-12T18:14:55,858 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3cc7b95394be4b1594453cc3296a7ba8 2024-11-12T18:14:55,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742095_1271 (size=12493) 2024-11-12T18:14:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/fe8e06f2203f4af08c56bf10e064c3ad is 50, key is test_row_0/B:col10/1731435295651/Put/seqid=0 2024-11-12T18:14:55,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742097_1273 (size=12493) 2024-11-12T18:14:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742098_1274 (size=12151) 2024-11-12T18:14:55,952 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/fe8e06f2203f4af08c56bf10e064c3ad 2024-11-12T18:14:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T18:14:55,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/f7653e93aa194ad99abdc9b23eab39ee is 50, key is test_row_0/C:col10/1731435295651/Put/seqid=0 2024-11-12T18:14:55,981 DEBUG [regionserver/9911683f163c:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-12T18:14:55,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:56,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742099_1275 (size=12151) 2024-11-12T18:14:56,007 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/f7653e93aa194ad99abdc9b23eab39ee 2024-11-12T18:14:56,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3cc7b95394be4b1594453cc3296a7ba8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3cc7b95394be4b1594453cc3296a7ba8 2024-11-12T18:14:56,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3cc7b95394be4b1594453cc3296a7ba8, entries=150, sequenceid=164, filesize=11.9 K 2024-11-12T18:14:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/fe8e06f2203f4af08c56bf10e064c3ad as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/fe8e06f2203f4af08c56bf10e064c3ad 2024-11-12T18:14:56,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/fe8e06f2203f4af08c56bf10e064c3ad, entries=150, sequenceid=164, filesize=11.9 K 2024-11-12T18:14:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/f7653e93aa194ad99abdc9b23eab39ee as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/f7653e93aa194ad99abdc9b23eab39ee 2024-11-12T18:14:56,043 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/f7653e93aa194ad99abdc9b23eab39ee, entries=150, sequenceid=164, filesize=11.9 K 2024-11-12T18:14:56,045 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=46.96 KB/48090 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 243ms, sequenceid=164, compaction requested=true 2024-11-12T18:14:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-12T18:14:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-12T18:14:56,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:56,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T18:14:56,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 399 msec 2024-11-12T18:14:56,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 408 msec 2024-11-12T18:14:56,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/06f8ba07f3a94d318833cd8769f1cfa8 is 50, key is test_row_1/A:col10/1731435296046/Put/seqid=0 2024-11-12T18:14:56,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742100_1276 (size=12147) 2024-11-12T18:14:56,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435296086, max timestamp=1731435296167Num uniques 859; 2024-11-12T18:14:56,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435296086, max timestamp=1731435296167Num uniques 859; 2024-11-12T18:14:56,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435296086, max timestamp=1731435296167Num uniques 859; 2024-11-12T18:14:56,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:56,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:56,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435356233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435356234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435356235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T18:14:56,270 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:14:56,271 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:14:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-12T18:14:56,274 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:14:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:14:56,275 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:14:56,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:14:56,308 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/ea8f608b6003449eb7d4f7d16ba4ffd6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ea8f608b6003449eb7d4f7d16ba4ffd6 2024-11-12T18:14:56,315 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into ea8f608b6003449eb7d4f7d16ba4ffd6(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:56,315 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:56,315 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=12, startTime=1731435295790; duration=0sec 2024-11-12T18:14:56,316 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:56,316 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:14:56,316 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T18:14:56,317 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T18:14:56,317 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:14:56,317 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,317 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5881c1b14d54400e9fe3afa0541377dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/23ff15ff18fb40ea8037bcddf524faa4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7618138ed0e64b579f7ed0d097944ced, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7bc757dd045549c49b5a31a2b1ce3ee5, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/f7653e93aa194ad99abdc9b23eab39ee] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=59.1 K 2024-11-12T18:14:56,318 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5881c1b14d54400e9fe3afa0541377dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435293843 2024-11-12T18:14:56,318 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ff15ff18fb40ea8037bcddf524faa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731435294237 2024-11-12T18:14:56,319 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7618138ed0e64b579f7ed0d097944ced, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731435294384 2024-11-12T18:14:56,319 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc757dd045549c49b5a31a2b1ce3ee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435295036 2024-11-12T18:14:56,320 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7653e93aa194ad99abdc9b23eab39ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731435295626 2024-11-12T18:14:56,332 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/7c6b955bdd9d4286a1dbc57b193bdc0e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7c6b955bdd9d4286a1dbc57b193bdc0e 2024-11-12T18:14:56,334 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#230 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:14:56,335 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0414d7b16bf84c6f9678d714106771fd is 50, key is test_row_0/C:col10/1731435295651/Put/seqid=0 2024-11-12T18:14:56,338 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 7c6b955bdd9d4286a1dbc57b193bdc0e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:56,338 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:56,338 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=12, startTime=1731435295790; duration=0sec 2024-11-12T18:14:56,338 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:56,338 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:14:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435356351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435356351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435356351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742101_1277 (size=12527) 2024-11-12T18:14:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:14:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435356400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,403 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_2' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435356410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,417 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/06f8ba07f3a94d318833cd8769f1cfa8 2024-11-12T18:14:56,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/2c927d8b7f7d45159649fb39e15dd655 is 50, key is test_row_1/B:col10/1731435296046/Put/seqid=0 2024-11-12T18:14:56,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742102_1278 (size=9757) 2024-11-12T18:14:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435356560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435356561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435356561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:14:56,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,768 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0414d7b16bf84c6f9678d714106771fd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0414d7b16bf84c6f9678d714106771fd 2024-11-12T18:14:56,782 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 0414d7b16bf84c6f9678d714106771fd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:56,782 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:56,782 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=11, startTime=1731435295790; duration=0sec 2024-11-12T18:14:56,782 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:56,782 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:14:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435356870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435356870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:56,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435356871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:56,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:56,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:56,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:56,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:14:56,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/2c927d8b7f7d45159649fb39e15dd655 2024-11-12T18:14:56,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/415be9eefe2a4084bfd2ac1db57600d2 is 50, key is test_row_1/C:col10/1731435296046/Put/seqid=0 2024-11-12T18:14:56,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742103_1279 (size=9757) 2024-11-12T18:14:57,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/415be9eefe2a4084bfd2ac1db57600d2 2024-11-12T18:14:57,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/06f8ba07f3a94d318833cd8769f1cfa8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06f8ba07f3a94d318833cd8769f1cfa8 2024-11-12T18:14:57,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06f8ba07f3a94d318833cd8769f1cfa8, entries=150, sequenceid=175, filesize=11.9 K 2024-11-12T18:14:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/2c927d8b7f7d45159649fb39e15dd655 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/2c927d8b7f7d45159649fb39e15dd655 2024-11-12T18:14:57,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/2c927d8b7f7d45159649fb39e15dd655, entries=100, sequenceid=175, filesize=9.5 K 2024-11-12T18:14:57,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/415be9eefe2a4084bfd2ac1db57600d2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/415be9eefe2a4084bfd2ac1db57600d2 2024-11-12T18:14:57,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/415be9eefe2a4084bfd2ac1db57600d2, entries=100, sequenceid=175, filesize=9.5 K 2024-11-12T18:14:57,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1334ms, sequenceid=175, compaction requested=true 2024-11-12T18:14:57,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:57,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:57,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:57,383 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:57,383 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:57,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:57,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:57,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:14:57,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:57,384 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:57,384 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:14:57,385 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,385 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ea8f608b6003449eb7d4f7d16ba4ffd6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3cc7b95394be4b1594453cc3296a7ba8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06f8ba07f3a94d318833cd8769f1cfa8] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=35.9 K 2024-11-12T18:14:57,386 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:57,386 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:14:57,386 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,386 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7c6b955bdd9d4286a1dbc57b193bdc0e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/fe8e06f2203f4af08c56bf10e064c3ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/2c927d8b7f7d45159649fb39e15dd655] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=33.6 K 2024-11-12T18:14:57,387 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6b955bdd9d4286a1dbc57b193bdc0e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435295036 2024-11-12T18:14:57,387 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea8f608b6003449eb7d4f7d16ba4ffd6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731435295036 2024-11-12T18:14:57,387 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting fe8e06f2203f4af08c56bf10e064c3ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731435295626 2024-11-12T18:14:57,388 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc7b95394be4b1594453cc3296a7ba8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731435295626 2024-11-12T18:14:57,388 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f8ba07f3a94d318833cd8769f1cfa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731435296035 2024-11-12T18:14:57,388 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c927d8b7f7d45159649fb39e15dd655, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731435296040 2024-11-12T18:14:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:57,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T18:14:57,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731435296222, max timestamp=1731435297390Num uniques 1018; 2024-11-12T18:14:57,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T18:14:57,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731435296222, max timestamp=1731435297390Num uniques 1018; 2024-11-12T18:14:57,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T18:14:57,400 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:14:57,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731435296222, max timestamp=1731435297390Num uniques 1018; 2024-11-12T18:14:57,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T18:14:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:57,405 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/eccdc6cdaf034d9e95dd673ae3fc0c5c is 50, key is test_row_0/B:col10/1731435295651/Put/seqid=0 2024-11-12T18:14:57,410 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:14:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:14:57,410 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/692b027b9afb4ee99e6060da43a1f05f is 50, key is test_row_0/A:col10/1731435295651/Put/seqid=0 2024-11-12T18:14:57,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8282d01d6e904202b4786f10e2a71693 is 50, key is test_row_0/A:col10/1731435296222/Put/seqid=0 2024-11-12T18:14:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742104_1280 (size=12595) 2024-11-12T18:14:57,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435297428, max timestamp=1731435297463Num uniques 859; 2024-11-12T18:14:57,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:57,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435297428, max timestamp=1731435297463Num uniques 859; 2024-11-12T18:14:57,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:57,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435297428, max timestamp=1731435297463Num uniques 859; 2024-11-12T18:14:57,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:14:57,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742105_1281 (size=12595) 2024-11-12T18:14:57,476 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/eccdc6cdaf034d9e95dd673ae3fc0c5c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/eccdc6cdaf034d9e95dd673ae3fc0c5c 2024-11-12T18:14:57,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742106_1282 (size=14541) 2024-11-12T18:14:57,484 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into eccdc6cdaf034d9e95dd673ae3fc0c5c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:57,484 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:57,484 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435297383; duration=0sec 2024-11-12T18:14:57,485 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:57,485 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:14:57,485 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:14:57,486 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:57,486 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:57,486 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. because compaction request was cancelled 2024-11-12T18:14:57,486 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:14:57,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435357531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:57,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435357533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:57,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435297521, max timestamp=1731435297540Num uniques 1068; 2024-11-12T18:14:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435297521, max timestamp=1731435297540Num uniques 1068; 2024-11-12T18:14:57,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T18:14:57,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T18:14:57,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435297521, max timestamp=1731435297540Num uniques 1068; 2024-11-12T18:14:57,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T18:14:57,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731435297638, max timestamp=1731435297669Num uniques 1027; 2024-11-12T18:14:57,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-12T18:14:57,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435297638, max timestamp=1731435297669Num uniques 1027; 2024-11-12T18:14:57,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-12T18:14:57,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435297638, max timestamp=1731435297669Num uniques 1027; 2024-11-12T18:14:57,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-12T18:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731435297704, max timestamp=1731435297726Num uniques 1136; 2024-11-12T18:14:57,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-12T18:14:57,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731435297704, max timestamp=1731435297726Num uniques 1009; 2024-11-12T18:14:57,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-11-12T18:14:57,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435297704, max timestamp=1731435297726Num uniques 1009; 2024-11-12T18:14:57,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-12T18:14:57,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731435297763, max timestamp=1731435297784Num uniques 1045; 2024-11-12T18:14:57,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-12T18:14:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731435297763, max timestamp=1731435297784Num uniques 1045; 2024-11-12T18:14:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-12T18:14:57,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731435297763, max timestamp=1731435297784Num uniques 1009; 2024-11-12T18:14:57,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-11-12T18:14:57,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,879 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/692b027b9afb4ee99e6060da43a1f05f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/692b027b9afb4ee99e6060da43a1f05f 2024-11-12T18:14:57,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8282d01d6e904202b4786f10e2a71693 2024-11-12T18:14:57,889 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 692b027b9afb4ee99e6060da43a1f05f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:57,889 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:57,889 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435297382; duration=0sec 2024-11-12T18:14:57,889 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:57,889 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:14:57,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/6b438c05bfd44fb396fd362884508b2b is 50, key is test_row_0/B:col10/1731435296222/Put/seqid=0 2024-11-12T18:14:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435357894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:57,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435357895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435357899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:57,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742107_1283 (size=12151) 2024-11-12T18:14:57,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:57,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435358010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435358010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435358011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435358220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435358221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435358230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/6b438c05bfd44fb396fd362884508b2b 2024-11-12T18:14:58,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/6931293504fc4d95be6eb0ecb9139ce2 is 50, key is test_row_0/C:col10/1731435296222/Put/seqid=0 2024-11-12T18:14:58,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742108_1284 (size=12151) 2024-11-12T18:14:58,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435358410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,415 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_2' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:14:58,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435358430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,436 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435358541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435358541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435358541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:58,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:14:58,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/6931293504fc4d95be6eb0ecb9139ce2 2024-11-12T18:14:58,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/8282d01d6e904202b4786f10e2a71693 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8282d01d6e904202b4786f10e2a71693 2024-11-12T18:14:58,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8282d01d6e904202b4786f10e2a71693, entries=200, sequenceid=217, filesize=14.2 K 2024-11-12T18:14:58,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/6b438c05bfd44fb396fd362884508b2b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6b438c05bfd44fb396fd362884508b2b 2024-11-12T18:14:58,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6b438c05bfd44fb396fd362884508b2b, entries=150, sequenceid=217, filesize=11.9 K 2024-11-12T18:14:58,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/6931293504fc4d95be6eb0ecb9139ce2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/6931293504fc4d95be6eb0ecb9139ce2 2024-11-12T18:14:58,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/6931293504fc4d95be6eb0ecb9139ce2, entries=150, sequenceid=217, filesize=11.9 K 2024-11-12T18:14:58,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~100.50 KB/102912, currentSize=149.83 KB/153430 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1406ms, sequenceid=217, compaction requested=true 2024-11-12T18:14:58,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:58,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:58,800 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:14:58,801 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:58,801 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:58,801 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. because compaction request was cancelled 2024-11-12T18:14:58,801 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:14:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:14:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:14:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:14:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:14:58,808 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:14:58,808 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:14:58,809 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:14:58,809 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:14:58,809 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. because compaction request was cancelled 2024-11-12T18:14:58,809 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:14:58,809 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:14:58,810 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:14:58,810 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,810 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0414d7b16bf84c6f9678d714106771fd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/415be9eefe2a4084bfd2ac1db57600d2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/6931293504fc4d95be6eb0ecb9139ce2] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=33.6 K 2024-11-12T18:14:58,810 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0414d7b16bf84c6f9678d714106771fd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731435295626 2024-11-12T18:14:58,811 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 415be9eefe2a4084bfd2ac1db57600d2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731435296040 2024-11-12T18:14:58,811 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6931293504fc4d95be6eb0ecb9139ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731435296222 2024-11-12T18:14:58,827 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:14:58,827 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/d18775c2211646a6837c62b5dec3b03c is 50, key is test_row_0/C:col10/1731435296222/Put/seqid=0 2024-11-12T18:14:58,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742109_1285 (size=12629) 2024-11-12T18:14:58,881 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/d18775c2211646a6837c62b5dec3b03c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d18775c2211646a6837c62b5dec3b03c 2024-11-12T18:14:58,888 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into d18775c2211646a6837c62b5dec3b03c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:14:58,888 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:58,888 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435298808; duration=0sec 2024-11-12T18:14:58,888 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:14:58,888 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:14:58,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:58,896 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=149.83 KB heapSize=421.84 KB 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:58,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:14:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/4f1283f37c6f44c7a6af5c93c1ec3473 is 50, key is test_row_0/A:col10/1731435297897/Put/seqid=0 2024-11-12T18:14:58,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742110_1286 (size=12301) 2024-11-12T18:14:58,935 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/4f1283f37c6f44c7a6af5c93c1ec3473 2024-11-12T18:14:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9a36972a46c9457a8a46936d6ff4f52a is 50, key is test_row_0/B:col10/1731435297897/Put/seqid=0 2024-11-12T18:14:59,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742111_1287 (size=12301) 2024-11-12T18:14:59,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9a36972a46c9457a8a46936d6ff4f52a 2024-11-12T18:14:59,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c96f3b66063b4ee1b297ffebb4ac667a is 50, key is test_row_0/C:col10/1731435297897/Put/seqid=0 2024-11-12T18:14:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:59,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:14:59,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742112_1288 (size=12301) 2024-11-12T18:14:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435359071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435359073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435359075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435359180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435359190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435359190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435359390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435359400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435359400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,460 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c96f3b66063b4ee1b297ffebb4ac667a 2024-11-12T18:14:59,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/4f1283f37c6f44c7a6af5c93c1ec3473 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/4f1283f37c6f44c7a6af5c93c1ec3473 2024-11-12T18:14:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/4f1283f37c6f44c7a6af5c93c1ec3473, entries=150, sequenceid=326, filesize=12.0 K 2024-11-12T18:14:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9a36972a46c9457a8a46936d6ff4f52a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9a36972a46c9457a8a46936d6ff4f52a 2024-11-12T18:14:59,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9a36972a46c9457a8a46936d6ff4f52a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-12T18:14:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c96f3b66063b4ee1b297ffebb4ac667a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c96f3b66063b4ee1b297ffebb4ac667a 2024-11-12T18:14:59,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c96f3b66063b4ee1b297ffebb4ac667a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-12T18:14:59,482 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~421.80 KB/431920, currentSize=33.54 KB/34350 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 586ms, sequenceid=326, compaction requested=true 2024-11-12T18:14:59,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:14:59,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:14:59,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-12T18:14:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-12T18:14:59,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T18:14:59,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2100 sec 2024-11-12T18:14:59,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 3.2180 sec 2024-11-12T18:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:14:59,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:14:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:14:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:14:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:14:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:14:59,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/a8a9b1a2c22046a2b1683c5aab1f7db8 is 50, key is test_row_0/A:col10/1731435299701/Put/seqid=0 2024-11-12T18:14:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742113_1289 (size=14741) 2024-11-12T18:14:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435359747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435299743, max timestamp=1731435299750Num uniques 859; 2024-11-12T18:14:59,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:59,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435299743, max timestamp=1731435299750Num uniques 859; 2024-11-12T18:14:59,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:59,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435299743, max timestamp=1731435299750Num uniques 859; 2024-11-12T18:14:59,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:14:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435359801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435359802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435359860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435359920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435359920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435360070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/a8a9b1a2c22046a2b1683c5aab1f7db8 2024-11-12T18:15:00,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435360130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435360130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/5bdef56108464be294d649cc9dbd0c45 is 50, key is test_row_0/B:col10/1731435299701/Put/seqid=0 2024-11-12T18:15:00,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742114_1290 (size=12301) 2024-11-12T18:15:00,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/5bdef56108464be294d649cc9dbd0c45 2024-11-12T18:15:00,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0f86fb8d46f943ec9f5b40ab1df051ee is 50, key is test_row_0/C:col10/1731435299701/Put/seqid=0 2024-11-12T18:15:00,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742115_1291 (size=12301) 2024-11-12T18:15:00,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0f86fb8d46f943ec9f5b40ab1df051ee 2024-11-12T18:15:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/a8a9b1a2c22046a2b1683c5aab1f7db8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a8a9b1a2c22046a2b1683c5aab1f7db8 2024-11-12T18:15:00,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a8a9b1a2c22046a2b1683c5aab1f7db8, entries=200, sequenceid=336, filesize=14.4 K 2024-11-12T18:15:00,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/5bdef56108464be294d649cc9dbd0c45 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5bdef56108464be294d649cc9dbd0c45 2024-11-12T18:15:00,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5bdef56108464be294d649cc9dbd0c45, entries=150, sequenceid=336, filesize=12.0 K 2024-11-12T18:15:00,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0f86fb8d46f943ec9f5b40ab1df051ee as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0f86fb8d46f943ec9f5b40ab1df051ee 2024-11-12T18:15:00,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0f86fb8d46f943ec9f5b40ab1df051ee, entries=150, sequenceid=336, filesize=12.0 K 2024-11-12T18:15:00,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 502ms, sequenceid=336, compaction requested=true 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:00,209 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:00,209 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:00,210 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54178 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:00,210 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:00,210 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:00,210 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:00,210 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,210 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,210 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/692b027b9afb4ee99e6060da43a1f05f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8282d01d6e904202b4786f10e2a71693, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/4f1283f37c6f44c7a6af5c93c1ec3473, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a8a9b1a2c22046a2b1683c5aab1f7db8] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=52.9 K 2024-11-12T18:15:00,210 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/eccdc6cdaf034d9e95dd673ae3fc0c5c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6b438c05bfd44fb396fd362884508b2b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9a36972a46c9457a8a46936d6ff4f52a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5bdef56108464be294d649cc9dbd0c45] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=48.2 K 2024-11-12T18:15:00,211 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 692b027b9afb4ee99e6060da43a1f05f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731435295651 2024-11-12T18:15:00,211 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting eccdc6cdaf034d9e95dd673ae3fc0c5c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731435295651 2024-11-12T18:15:00,211 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8282d01d6e904202b4786f10e2a71693, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731435296222 2024-11-12T18:15:00,211 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b438c05bfd44fb396fd362884508b2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731435296222 2024-11-12T18:15:00,211 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1283f37c6f44c7a6af5c93c1ec3473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731435297882 2024-11-12T18:15:00,211 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a36972a46c9457a8a46936d6ff4f52a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731435297882 2024-11-12T18:15:00,212 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a9b1a2c22046a2b1683c5aab1f7db8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731435299065 2024-11-12T18:15:00,212 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bdef56108464be294d649cc9dbd0c45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731435299067 2024-11-12T18:15:00,223 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:00,224 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/07a419ee28314992b59c8809ab3c4ae5 is 50, key is test_row_0/B:col10/1731435299701/Put/seqid=0 2024-11-12T18:15:00,227 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:00,228 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c6caea84f34f44ecb2dfdc4449164580 is 50, key is test_row_0/A:col10/1731435299701/Put/seqid=0 2024-11-12T18:15:00,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742116_1292 (size=12881) 2024-11-12T18:15:00,256 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/07a419ee28314992b59c8809ab3c4ae5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/07a419ee28314992b59c8809ab3c4ae5 2024-11-12T18:15:00,263 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 07a419ee28314992b59c8809ab3c4ae5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:00,263 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:00,264 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=12, startTime=1731435300209; duration=0sec 2024-11-12T18:15:00,264 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:00,264 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:00,264 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:00,265 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:00,265 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:00,266 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,266 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d18775c2211646a6837c62b5dec3b03c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c96f3b66063b4ee1b297ffebb4ac667a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0f86fb8d46f943ec9f5b40ab1df051ee] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.4 K 2024-11-12T18:15:00,266 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18775c2211646a6837c62b5dec3b03c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731435296222 2024-11-12T18:15:00,267 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c96f3b66063b4ee1b297ffebb4ac667a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731435297882 2024-11-12T18:15:00,267 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f86fb8d46f943ec9f5b40ab1df051ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731435299067 2024-11-12T18:15:00,286 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:00,287 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0768a44d0fb94b86a60b6cdbd678c75b is 50, key is test_row_0/C:col10/1731435299701/Put/seqid=0 2024-11-12T18:15:00,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742117_1293 (size=12881) 2024-11-12T18:15:00,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742118_1294 (size=12881) 2024-11-12T18:15:00,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T18:15:00,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:00,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:15:00,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:00,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:15:00,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:00,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T18:15:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:00,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/a06f9bc52c0947d98522eed7742873c0 is 50, key is test_row_0/A:col10/1731435299791/Put/seqid=0 2024-11-12T18:15:00,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742119_1295 (size=14741) 2024-11-12T18:15:00,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/a06f9bc52c0947d98522eed7742873c0 2024-11-12T18:15:00,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/a30dacc3743749b6825c9a17984ddd2c is 50, key is test_row_0/B:col10/1731435299791/Put/seqid=0 2024-11-12T18:15:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742120_1296 (size=12301) 2024-11-12T18:15:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435360427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T18:15:00,441 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:00,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-12T18:15:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T18:15:00,444 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:00,444 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:00,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435360440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435360440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435360540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T18:15:00,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T18:15:00,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:00,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,695 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c6caea84f34f44ecb2dfdc4449164580 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c6caea84f34f44ecb2dfdc4449164580 2024-11-12T18:15:00,700 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into c6caea84f34f44ecb2dfdc4449164580(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:00,700 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:00,700 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=12, startTime=1731435300209; duration=0sec 2024-11-12T18:15:00,700 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:00,701 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:00,722 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0768a44d0fb94b86a60b6cdbd678c75b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0768a44d0fb94b86a60b6cdbd678c75b 2024-11-12T18:15:00,728 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 0768a44d0fb94b86a60b6cdbd678c75b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:00,728 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:00,728 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435300209; duration=0sec 2024-11-12T18:15:00,728 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:00,728 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:00,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T18:15:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435360750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T18:15:00,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/a30dacc3743749b6825c9a17984ddd2c 2024-11-12T18:15:00,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/e930fdde47414a55ab8f469685095b74 is 50, key is test_row_0/C:col10/1731435299791/Put/seqid=0 2024-11-12T18:15:00,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742121_1297 (size=12301) 2024-11-12T18:15:00,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T18:15:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:00,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:00,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435360952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:00,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435360960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:01,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T18:15:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435361060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T18:15:01,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T18:15:01,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:01,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/e930fdde47414a55ab8f469685095b74 2024-11-12T18:15:01,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/a06f9bc52c0947d98522eed7742873c0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a06f9bc52c0947d98522eed7742873c0 2024-11-12T18:15:01,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a06f9bc52c0947d98522eed7742873c0, entries=200, sequenceid=375, filesize=14.4 K 2024-11-12T18:15:01,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/a30dacc3743749b6825c9a17984ddd2c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a30dacc3743749b6825c9a17984ddd2c 2024-11-12T18:15:01,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a30dacc3743749b6825c9a17984ddd2c, entries=150, sequenceid=375, filesize=12.0 K 2024-11-12T18:15:01,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/e930fdde47414a55ab8f469685095b74 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/e930fdde47414a55ab8f469685095b74 2024-11-12T18:15:01,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/e930fdde47414a55ab8f469685095b74, entries=150, sequenceid=375, filesize=12.0 K 2024-11-12T18:15:01,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 902ms, sequenceid=375, compaction requested=false 2024-11-12T18:15:01,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:01,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T18:15:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,370 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T18:15:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:01,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/02f5ac1f602a4993ac454109e00dbdaf is 50, key is test_row_1/A:col10/1731435300423/Put/seqid=0 2024-11-12T18:15:01,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742122_1298 (size=9857) 2024-11-12T18:15:01,392 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/02f5ac1f602a4993ac454109e00dbdaf 2024-11-12T18:15:01,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8a290b86671749269e7d2e9e3f952b00 is 50, key is test_row_1/B:col10/1731435300423/Put/seqid=0 2024-11-12T18:15:01,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742123_1299 (size=9857) 2024-11-12T18:15:01,426 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8a290b86671749269e7d2e9e3f952b00 2024-11-12T18:15:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/40e273a7266045c48e2bacfe4438a253 is 50, key is test_row_1/C:col10/1731435300423/Put/seqid=0 2024-11-12T18:15:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742124_1300 (size=9857) 2024-11-12T18:15:01,448 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/40e273a7266045c48e2bacfe4438a253 2024-11-12T18:15:01,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/02f5ac1f602a4993ac454109e00dbdaf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/02f5ac1f602a4993ac454109e00dbdaf 2024-11-12T18:15:01,479 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/02f5ac1f602a4993ac454109e00dbdaf, entries=100, sequenceid=386, filesize=9.6 K 2024-11-12T18:15:01,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8a290b86671749269e7d2e9e3f952b00 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8a290b86671749269e7d2e9e3f952b00 2024-11-12T18:15:01,487 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8a290b86671749269e7d2e9e3f952b00, entries=100, sequenceid=386, filesize=9.6 K 2024-11-12T18:15:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/40e273a7266045c48e2bacfe4438a253 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/40e273a7266045c48e2bacfe4438a253 2024-11-12T18:15:01,497 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/40e273a7266045c48e2bacfe4438a253, entries=100, sequenceid=386, filesize=9.6 K 2024-11-12T18:15:01,498 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 128ms, sequenceid=386, compaction requested=true 2024-11-12T18:15:01,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:01,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-12T18:15:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-12T18:15:01,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-12T18:15:01,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0560 sec 2024-11-12T18:15:01,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0620 sec 2024-11-12T18:15:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T18:15:01,581 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:01,582 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-12T18:15:01,585 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:01,586 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:01,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:01,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:15:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:01,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e06cce1857d544268f5afdcec41b04c0 is 50, key is test_row_0/A:col10/1731435301581/Put/seqid=0 2024-11-12T18:15:01,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742125_1301 (size=14741) 2024-11-12T18:15:01,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e06cce1857d544268f5afdcec41b04c0 2024-11-12T18:15:01,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/71841e4fb65d4b5cb963d61c541296a2 is 50, key is test_row_0/B:col10/1731435301581/Put/seqid=0 2024-11-12T18:15:01,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742126_1302 (size=12301) 2024-11-12T18:15:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435301685, max timestamp=1731435301702Num uniques 859; 2024-11-12T18:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435301685, max timestamp=1731435301702Num uniques 859; 2024-11-12T18:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:01,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435301685, max timestamp=1731435301702Num uniques 859; 2024-11-12T18:15:01,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:01,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T18:15:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435361832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:01,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T18:15:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:01,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:01,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:01,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435361940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:01,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435361971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:01,978 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435361980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:01,985 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_2' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:02,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/71841e4fb65d4b5cb963d61c541296a2 2024-11-12T18:15:02,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c2067569ac6841e7bb7b0596ecd00b76 is 50, key is test_row_0/C:col10/1731435301581/Put/seqid=0 2024-11-12T18:15:02,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742127_1303 (size=12301) 2024-11-12T18:15:02,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T18:15:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435362150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:02,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T18:15:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:02,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T18:15:02,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:02,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435362440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:02,447 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_2' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:02,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c2067569ac6841e7bb7b0596ecd00b76 2024-11-12T18:15:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e06cce1857d544268f5afdcec41b04c0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e06cce1857d544268f5afdcec41b04c0 2024-11-12T18:15:02,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:02,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435362461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:02,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e06cce1857d544268f5afdcec41b04c0, entries=200, sequenceid=396, filesize=14.4 K 2024-11-12T18:15:02,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/71841e4fb65d4b5cb963d61c541296a2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/71841e4fb65d4b5cb963d61c541296a2 2024-11-12T18:15:02,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435362480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:02,483 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:02,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/71841e4fb65d4b5cb963d61c541296a2, entries=150, sequenceid=396, filesize=12.0 K 2024-11-12T18:15:02,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c2067569ac6841e7bb7b0596ecd00b76 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c2067569ac6841e7bb7b0596ecd00b76 2024-11-12T18:15:02,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c2067569ac6841e7bb7b0596ecd00b76, entries=150, sequenceid=396, filesize=12.0 K 2024-11-12T18:15:02,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 890ms, sequenceid=396, compaction requested=true 2024-11-12T18:15:02,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:02,494 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:02,494 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:02,495 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52220 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:02,495 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:02,495 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,495 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c6caea84f34f44ecb2dfdc4449164580, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a06f9bc52c0947d98522eed7742873c0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/02f5ac1f602a4993ac454109e00dbdaf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e06cce1857d544268f5afdcec41b04c0] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=51.0 K 2024-11-12T18:15:02,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:02,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:02,496 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,496 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/07a419ee28314992b59c8809ab3c4ae5, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a30dacc3743749b6825c9a17984ddd2c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8a290b86671749269e7d2e9e3f952b00, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/71841e4fb65d4b5cb963d61c541296a2] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=46.2 K 2024-11-12T18:15:02,496 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6caea84f34f44ecb2dfdc4449164580, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731435299067 2024-11-12T18:15:02,497 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a419ee28314992b59c8809ab3c4ae5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731435299067 2024-11-12T18:15:02,497 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting a06f9bc52c0947d98522eed7742873c0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731435299778 2024-11-12T18:15:02,497 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a30dacc3743749b6825c9a17984ddd2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731435299778 2024-11-12T18:15:02,497 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f5ac1f602a4993ac454109e00dbdaf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731435300417 2024-11-12T18:15:02,497 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a290b86671749269e7d2e9e3f952b00, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731435300417 2024-11-12T18:15:02,497 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06cce1857d544268f5afdcec41b04c0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731435301581 2024-11-12T18:15:02,498 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 71841e4fb65d4b5cb963d61c541296a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731435301581 2024-11-12T18:15:02,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,512 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:02,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:02,516 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#257 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:02,516 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e4ebb80f52924d8895551a63da9e8748 is 50, key is test_row_0/A:col10/1731435301581/Put/seqid=0 2024-11-12T18:15:02,519 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:02,520 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/40d7c2838f4d4a5eb64f7481e8b04314 is 50, key is test_row_0/B:col10/1731435301581/Put/seqid=0 2024-11-12T18:15:02,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/db7bfeff7fe848b98babe8fcdee11722 is 50, key is test_row_0/A:col10/1731435301802/Put/seqid=0 2024-11-12T18:15:02,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742128_1304 (size=13017) 2024-11-12T18:15:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742129_1305 (size=13017) 2024-11-12T18:15:02,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742130_1306 (size=12301) 2024-11-12T18:15:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:02,936 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e4ebb80f52924d8895551a63da9e8748 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4ebb80f52924d8895551a63da9e8748 2024-11-12T18:15:02,939 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/40d7c2838f4d4a5eb64f7481e8b04314 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/40d7c2838f4d4a5eb64f7481e8b04314 2024-11-12T18:15:02,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/db7bfeff7fe848b98babe8fcdee11722 2024-11-12T18:15:02,943 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into e4ebb80f52924d8895551a63da9e8748(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:02,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:02,943 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=12, startTime=1731435302493; duration=0sec 2024-11-12T18:15:02,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:02,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:02,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:02,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:02,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:02,946 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:02,946 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0768a44d0fb94b86a60b6cdbd678c75b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/e930fdde47414a55ab8f469685095b74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/40e273a7266045c48e2bacfe4438a253, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c2067569ac6841e7bb7b0596ecd00b76] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=46.2 K 2024-11-12T18:15:02,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0768a44d0fb94b86a60b6cdbd678c75b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731435299067 2024-11-12T18:15:02,948 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 40d7c2838f4d4a5eb64f7481e8b04314(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:02,948 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:02,948 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=12, startTime=1731435302494; duration=0sec 2024-11-12T18:15:02,948 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:02,948 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:02,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e930fdde47414a55ab8f469685095b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731435299778 2024-11-12T18:15:02,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e273a7266045c48e2bacfe4438a253, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731435300417 2024-11-12T18:15:02,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2067569ac6841e7bb7b0596ecd00b76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731435301581 2024-11-12T18:15:02,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/3e04529b3c1348e2973c8bb8ecfbcf7a is 50, key is test_row_0/B:col10/1731435301802/Put/seqid=0 2024-11-12T18:15:02,960 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:02,961 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/64cc578a94d443f78b31a98941945764 is 50, key is test_row_0/C:col10/1731435301581/Put/seqid=0 2024-11-12T18:15:02,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742131_1307 (size=12301) 2024-11-12T18:15:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:02,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:02,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742132_1308 (size=13017) 2024-11-12T18:15:02,982 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/64cc578a94d443f78b31a98941945764 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/64cc578a94d443f78b31a98941945764 2024-11-12T18:15:02,988 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 64cc578a94d443f78b31a98941945764(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:02,989 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:02,989 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=12, startTime=1731435302494; duration=0sec 2024-11-12T18:15:02,989 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:02,989 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:03,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435363004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:03,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435363120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:03,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435363331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:03,371 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/3e04529b3c1348e2973c8bb8ecfbcf7a 2024-11-12T18:15:03,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5ed12be769f4471e81748c4f54ca7365 is 50, key is test_row_0/C:col10/1731435301802/Put/seqid=0 2024-11-12T18:15:03,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742133_1309 (size=12301) 2024-11-12T18:15:03,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435363640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:03,785 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5ed12be769f4471e81748c4f54ca7365 2024-11-12T18:15:03,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/db7bfeff7fe848b98babe8fcdee11722 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/db7bfeff7fe848b98babe8fcdee11722 2024-11-12T18:15:03,795 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/db7bfeff7fe848b98babe8fcdee11722, entries=150, sequenceid=433, filesize=12.0 K 2024-11-12T18:15:03,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/3e04529b3c1348e2973c8bb8ecfbcf7a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/3e04529b3c1348e2973c8bb8ecfbcf7a 2024-11-12T18:15:03,800 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/3e04529b3c1348e2973c8bb8ecfbcf7a, entries=150, sequenceid=433, filesize=12.0 K 2024-11-12T18:15:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5ed12be769f4471e81748c4f54ca7365 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5ed12be769f4471e81748c4f54ca7365 2024-11-12T18:15:03,805 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5ed12be769f4471e81748c4f54ca7365, entries=150, sequenceid=433, filesize=12.0 K 2024-11-12T18:15:03,806 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1294ms, sequenceid=433, compaction requested=false 2024-11-12T18:15:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-12T18:15:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-12T18:15:03,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T18:15:03,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2220 sec 2024-11-12T18:15:03,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2300 sec 2024-11-12T18:15:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:03,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:03,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/9715896d325a48d99dd01bdcf8f9f3f6 is 50, key is test_row_0/A:col10/1731435302997/Put/seqid=0 2024-11-12T18:15:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742134_1310 (size=12301) 2024-11-12T18:15:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435304027, max timestamp=1731435304039Num uniques 859; 2024-11-12T18:15:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435304027, max timestamp=1731435304039Num uniques 859; 2024-11-12T18:15:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435304027, max timestamp=1731435304039Num uniques 859; 2024-11-12T18:15:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435364081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435364082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435364150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435364190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435364190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/9715896d325a48d99dd01bdcf8f9f3f6 2024-11-12T18:15:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435364400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435364400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/5e1de8a5ed3d401c8cfa7622170e4227 is 50, key is test_row_0/B:col10/1731435302997/Put/seqid=0 2024-11-12T18:15:04,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742135_1311 (size=12301) 2024-11-12T18:15:04,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/5e1de8a5ed3d401c8cfa7622170e4227 2024-11-12T18:15:04,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/8a0cdf3c6da6400a956110c371db6636 is 50, key is test_row_0/C:col10/1731435302997/Put/seqid=0 2024-11-12T18:15:04,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742136_1312 (size=12301) 2024-11-12T18:15:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435364710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435364710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:04,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/8a0cdf3c6da6400a956110c371db6636 2024-11-12T18:15:04,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/9715896d325a48d99dd01bdcf8f9f3f6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9715896d325a48d99dd01bdcf8f9f3f6 2024-11-12T18:15:04,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9715896d325a48d99dd01bdcf8f9f3f6, entries=150, sequenceid=447, filesize=12.0 K 2024-11-12T18:15:04,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/5e1de8a5ed3d401c8cfa7622170e4227 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5e1de8a5ed3d401c8cfa7622170e4227 2024-11-12T18:15:04,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5e1de8a5ed3d401c8cfa7622170e4227, entries=150, sequenceid=447, filesize=12.0 K 2024-11-12T18:15:04,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/8a0cdf3c6da6400a956110c371db6636 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8a0cdf3c6da6400a956110c371db6636 2024-11-12T18:15:04,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8a0cdf3c6da6400a956110c371db6636, entries=150, sequenceid=447, filesize=12.0 K 2024-11-12T18:15:04,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 855ms, sequenceid=447, compaction requested=true 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:04,849 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:04,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:04,849 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:04,852 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:04,852 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:04,852 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:04,852 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:04,852 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:04,852 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:04,853 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4ebb80f52924d8895551a63da9e8748, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/db7bfeff7fe848b98babe8fcdee11722, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9715896d325a48d99dd01bdcf8f9f3f6] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.7 K 2024-11-12T18:15:04,853 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/40d7c2838f4d4a5eb64f7481e8b04314, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/3e04529b3c1348e2973c8bb8ecfbcf7a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5e1de8a5ed3d401c8cfa7622170e4227] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.7 K 2024-11-12T18:15:04,853 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ebb80f52924d8895551a63da9e8748, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731435301581 2024-11-12T18:15:04,853 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d7c2838f4d4a5eb64f7481e8b04314, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731435301581 2024-11-12T18:15:04,854 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e04529b3c1348e2973c8bb8ecfbcf7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731435301802 2024-11-12T18:15:04,854 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7bfeff7fe848b98babe8fcdee11722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731435301802 2024-11-12T18:15:04,854 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1de8a5ed3d401c8cfa7622170e4227, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731435302997 2024-11-12T18:15:04,854 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9715896d325a48d99dd01bdcf8f9f3f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731435302997 2024-11-12T18:15:04,867 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:04,867 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/116eb8540cd64161b6fbfd07e585064b is 50, key is test_row_0/B:col10/1731435302997/Put/seqid=0 2024-11-12T18:15:04,869 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#266 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:04,870 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c4aff2c167bd4898ba8d1b514fcc1ab7 is 50, key is test_row_0/A:col10/1731435302997/Put/seqid=0 2024-11-12T18:15:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742138_1314 (size=13119) 2024-11-12T18:15:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742137_1313 (size=13119) 2024-11-12T18:15:04,891 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/116eb8540cd64161b6fbfd07e585064b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/116eb8540cd64161b6fbfd07e585064b 2024-11-12T18:15:04,897 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 116eb8540cd64161b6fbfd07e585064b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:04,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:04,898 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435304849; duration=0sec 2024-11-12T18:15:04,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:04,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:04,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:04,899 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:04,899 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:04,899 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:04,899 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/64cc578a94d443f78b31a98941945764, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5ed12be769f4471e81748c4f54ca7365, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8a0cdf3c6da6400a956110c371db6636] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.7 K 2024-11-12T18:15:04,899 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 64cc578a94d443f78b31a98941945764, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731435301581 2024-11-12T18:15:04,900 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed12be769f4471e81748c4f54ca7365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731435301802 2024-11-12T18:15:04,900 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0cdf3c6da6400a956110c371db6636, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731435302997 2024-11-12T18:15:04,908 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:04,909 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/21de62ef3df64059acb0e7b591f065af is 50, key is test_row_0/C:col10/1731435302997/Put/seqid=0 2024-11-12T18:15:04,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742139_1315 (size=13119) 2024-11-12T18:15:04,922 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/21de62ef3df64059acb0e7b591f065af as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/21de62ef3df64059acb0e7b591f065af 2024-11-12T18:15:04,928 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 21de62ef3df64059acb0e7b591f065af(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:04,928 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:04,928 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435304849; duration=0sec 2024-11-12T18:15:04,928 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:04,929 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:05,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T18:15:05,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:05,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:05,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:05,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:05,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:05,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:05,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/ae07ab7269954e1ba2a18ccdb4414c0d is 50, key is test_row_0/A:col10/1731435305162/Put/seqid=0 2024-11-12T18:15:05,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742140_1316 (size=14741) 2024-11-12T18:15:05,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/ae07ab7269954e1ba2a18ccdb4414c0d 2024-11-12T18:15:05,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/a8e0904ca70840c8bd0b6d093d2bb8a4 is 50, key is test_row_0/B:col10/1731435305162/Put/seqid=0 2024-11-12T18:15:05,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742141_1317 (size=12301) 2024-11-12T18:15:05,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/a8e0904ca70840c8bd0b6d093d2bb8a4 2024-11-12T18:15:05,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0803e61f4d0649c3bbcfa55cba9d9158 is 50, key is test_row_0/C:col10/1731435305162/Put/seqid=0 2024-11-12T18:15:05,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742142_1318 (size=12301) 2024-11-12T18:15:05,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0803e61f4d0649c3bbcfa55cba9d9158 2024-11-12T18:15:05,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435365212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/ae07ab7269954e1ba2a18ccdb4414c0d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ae07ab7269954e1ba2a18ccdb4414c0d 2024-11-12T18:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:05,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ae07ab7269954e1ba2a18ccdb4414c0d, entries=200, sequenceid=485, filesize=14.4 K 2024-11-12T18:15:05,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/a8e0904ca70840c8bd0b6d093d2bb8a4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a8e0904ca70840c8bd0b6d093d2bb8a4 2024-11-12T18:15:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435365220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:05,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a8e0904ca70840c8bd0b6d093d2bb8a4, entries=150, sequenceid=485, filesize=12.0 K 2024-11-12T18:15:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0803e61f4d0649c3bbcfa55cba9d9158 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0803e61f4d0649c3bbcfa55cba9d9158 2024-11-12T18:15:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435365230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:05,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0803e61f4d0649c3bbcfa55cba9d9158, entries=150, sequenceid=485, filesize=12.0 K 2024-11-12T18:15:05,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 71ms, sequenceid=485, compaction requested=false 2024-11-12T18:15:05,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:05,292 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c4aff2c167bd4898ba8d1b514fcc1ab7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c4aff2c167bd4898ba8d1b514fcc1ab7 2024-11-12T18:15:05,301 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into c4aff2c167bd4898ba8d1b514fcc1ab7(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:05,301 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:05,301 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435304849; duration=0sec 2024-11-12T18:15:05,301 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:05,301 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:05,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:05,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/61172cc07daf4818b41e77883c72ad17 is 50, key is test_row_0/A:col10/1731435305189/Put/seqid=0 2024-11-12T18:15:05,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742143_1319 (size=12301) 2024-11-12T18:15:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435305401, max timestamp=1731435305424Num uniques 859; 2024-11-12T18:15:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435305401, max timestamp=1731435305424Num uniques 859; 2024-11-12T18:15:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435305401, max timestamp=1731435305424Num uniques 859; 2024-11-12T18:15:05,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:05,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435365524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435365640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T18:15:05,740 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:05,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-12T18:15:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T18:15:05,743 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:05,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/61172cc07daf4818b41e77883c72ad17 2024-11-12T18:15:05,744 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:05,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:05,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/dbe0d499bd124181b3467c47ee792bf4 is 50, key is test_row_0/B:col10/1731435305189/Put/seqid=0 2024-11-12T18:15:05,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742144_1320 (size=12301) 2024-11-12T18:15:05,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/dbe0d499bd124181b3467c47ee792bf4 2024-11-12T18:15:05,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c84bff579f7348398f2179a266cf4ec9 is 50, key is test_row_0/C:col10/1731435305189/Put/seqid=0 2024-11-12T18:15:05,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742145_1321 (size=12301) 2024-11-12T18:15:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T18:15:05,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435365850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:05,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T18:15:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:05,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:05,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:05,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:06,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T18:15:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:06,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T18:15:06,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435366160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c84bff579f7348398f2179a266cf4ec9 2024-11-12T18:15:06,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/61172cc07daf4818b41e77883c72ad17 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/61172cc07daf4818b41e77883c72ad17 2024-11-12T18:15:06,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/61172cc07daf4818b41e77883c72ad17, entries=150, sequenceid=497, filesize=12.0 K 2024-11-12T18:15:06,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/dbe0d499bd124181b3467c47ee792bf4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/dbe0d499bd124181b3467c47ee792bf4 2024-11-12T18:15:06,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/dbe0d499bd124181b3467c47ee792bf4, entries=150, sequenceid=497, filesize=12.0 K 2024-11-12T18:15:06,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c84bff579f7348398f2179a266cf4ec9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c84bff579f7348398f2179a266cf4ec9 2024-11-12T18:15:06,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c84bff579f7348398f2179a266cf4ec9, entries=150, sequenceid=497, filesize=12.0 K 2024-11-12T18:15:06,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 859ms, sequenceid=497, compaction requested=true 2024-11-12T18:15:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:06,192 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:06,192 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:06,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:06,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:06,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:06,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:06,194 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:06,194 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:06,194 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:06,194 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/116eb8540cd64161b6fbfd07e585064b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a8e0904ca70840c8bd0b6d093d2bb8a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/dbe0d499bd124181b3467c47ee792bf4] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.8 K 2024-11-12T18:15:06,194 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:06,194 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:06,194 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:06,194 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c4aff2c167bd4898ba8d1b514fcc1ab7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ae07ab7269954e1ba2a18ccdb4414c0d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/61172cc07daf4818b41e77883c72ad17] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=39.2 K 2024-11-12T18:15:06,194 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 116eb8540cd64161b6fbfd07e585064b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731435302997 2024-11-12T18:15:06,195 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e0904ca70840c8bd0b6d093d2bb8a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731435304068 2024-11-12T18:15:06,195 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4aff2c167bd4898ba8d1b514fcc1ab7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731435302997 2024-11-12T18:15:06,195 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting dbe0d499bd124181b3467c47ee792bf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731435305189 2024-11-12T18:15:06,195 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae07ab7269954e1ba2a18ccdb4414c0d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731435304068 2024-11-12T18:15:06,196 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61172cc07daf4818b41e77883c72ad17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731435305189 2024-11-12T18:15:06,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T18:15:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:06,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-12T18:15:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:06,206 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:06,207 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/16afd45b97d149d7a84eccea6cc0a585 is 50, key is test_row_0/B:col10/1731435305189/Put/seqid=0 2024-11-12T18:15:06,208 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:06,209 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/eb5f7c50e33144a2802ca46c7e52a24a is 50, key is test_row_0/A:col10/1731435305189/Put/seqid=0 2024-11-12T18:15:06,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3f5cd3dfae564e83a61a1b2e486a0a30 is 50, key is test_row_0/A:col10/1731435305480/Put/seqid=0 2024-11-12T18:15:06,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742147_1323 (size=13221) 2024-11-12T18:15:06,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:06,251 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/eb5f7c50e33144a2802ca46c7e52a24a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb5f7c50e33144a2802ca46c7e52a24a 2024-11-12T18:15:06,257 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into eb5f7c50e33144a2802ca46c7e52a24a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:06,257 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:06,257 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435306192; duration=0sec 2024-11-12T18:15:06,257 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:06,257 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:06,257 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:06,258 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:06,258 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:06,258 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:06,258 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/21de62ef3df64059acb0e7b591f065af, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0803e61f4d0649c3bbcfa55cba9d9158, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c84bff579f7348398f2179a266cf4ec9] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.8 K 2024-11-12T18:15:06,258 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21de62ef3df64059acb0e7b591f065af, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731435302997 2024-11-12T18:15:06,259 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0803e61f4d0649c3bbcfa55cba9d9158, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731435304068 2024-11-12T18:15:06,259 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c84bff579f7348398f2179a266cf4ec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731435305189 2024-11-12T18:15:06,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742146_1322 (size=13221) 2024-11-12T18:15:06,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742148_1324 (size=12301) 2024-11-12T18:15:06,270 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:06,271 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/1fadc627e8ed41eb8982aee6a57e576f is 50, key is test_row_0/C:col10/1731435305189/Put/seqid=0 2024-11-12T18:15:06,272 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/16afd45b97d149d7a84eccea6cc0a585 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/16afd45b97d149d7a84eccea6cc0a585 2024-11-12T18:15:06,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742149_1325 (size=13221) 2024-11-12T18:15:06,278 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 16afd45b97d149d7a84eccea6cc0a585(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:06,278 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:06,278 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435306192; duration=0sec 2024-11-12T18:15:06,279 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:06,279 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435366272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435366273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T18:15:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435366391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435366391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435366600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435366601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3f5cd3dfae564e83a61a1b2e486a0a30 2024-11-12T18:15:06,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435366670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/15ac48727aa743f4870844adb60cd3ba is 50, key is test_row_0/B:col10/1731435305480/Put/seqid=0 2024-11-12T18:15:06,679 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/1fadc627e8ed41eb8982aee6a57e576f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/1fadc627e8ed41eb8982aee6a57e576f 2024-11-12T18:15:06,684 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 1fadc627e8ed41eb8982aee6a57e576f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:06,684 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:06,684 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435306193; duration=0sec 2024-11-12T18:15:06,684 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:06,684 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:06,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742150_1326 (size=12301) 2024-11-12T18:15:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T18:15:06,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435366910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435366911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:07,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/15ac48727aa743f4870844adb60cd3ba 2024-11-12T18:15:07,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/584e64b3b4894d03b9434f165cbc6c71 is 50, key is test_row_0/C:col10/1731435305480/Put/seqid=0 2024-11-12T18:15:07,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742151_1327 (size=12301) 2024-11-12T18:15:07,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435367420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:07,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435367420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:07,503 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/584e64b3b4894d03b9434f165cbc6c71 2024-11-12T18:15:07,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3f5cd3dfae564e83a61a1b2e486a0a30 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3f5cd3dfae564e83a61a1b2e486a0a30 2024-11-12T18:15:07,512 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3f5cd3dfae564e83a61a1b2e486a0a30, entries=150, sequenceid=533, filesize=12.0 K 2024-11-12T18:15:07,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/15ac48727aa743f4870844adb60cd3ba as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/15ac48727aa743f4870844adb60cd3ba 2024-11-12T18:15:07,516 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/15ac48727aa743f4870844adb60cd3ba, entries=150, sequenceid=533, filesize=12.0 K 2024-11-12T18:15:07,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/584e64b3b4894d03b9434f165cbc6c71 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/584e64b3b4894d03b9434f165cbc6c71 2024-11-12T18:15:07,521 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/584e64b3b4894d03b9434f165cbc6c71, entries=150, sequenceid=533, filesize=12.0 K 2024-11-12T18:15:07,522 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1317ms, sequenceid=533, compaction requested=false 2024-11-12T18:15:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-12T18:15:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-12T18:15:07,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T18:15:07,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-11-12T18:15:07,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7860 sec 2024-11-12T18:15:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:07,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T18:15:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:07,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c5de1489a47a41fd92645a140905d855 is 50, key is test_row_0/A:col10/1731435306271/Put/seqid=0 2024-11-12T18:15:07,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742152_1328 (size=14741) 2024-11-12T18:15:07,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c5de1489a47a41fd92645a140905d855 2024-11-12T18:15:07,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/e775fe114c4e4f70b1def1209b43b233 is 50, key is test_row_0/B:col10/1731435306271/Put/seqid=0 2024-11-12T18:15:07,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742153_1329 (size=12301) 2024-11-12T18:15:07,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435367777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:07,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435367890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T18:15:07,900 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:07,901 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-12T18:15:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T18:15:07,903 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:07,904 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:07,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T18:15:08,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T18:15:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:08,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435368100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:08,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/e775fe114c4e4f70b1def1209b43b233 2024-11-12T18:15:08,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0aafd3e99ddf42da80b19ee2187bd0a1 is 50, key is test_row_0/C:col10/1731435306271/Put/seqid=0 2024-11-12T18:15:08,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742154_1330 (size=12301) 2024-11-12T18:15:08,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T18:15:08,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435368410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435368441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435368441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:08,446 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:08,447 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:08,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T18:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:08,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0aafd3e99ddf42da80b19ee2187bd0a1 2024-11-12T18:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:08,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/c5de1489a47a41fd92645a140905d855 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c5de1489a47a41fd92645a140905d855 2024-11-12T18:15:08,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c5de1489a47a41fd92645a140905d855, entries=200, sequenceid=548, filesize=14.4 K 2024-11-12T18:15:08,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/e775fe114c4e4f70b1def1209b43b233 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/e775fe114c4e4f70b1def1209b43b233 2024-11-12T18:15:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T18:15:08,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/e775fe114c4e4f70b1def1209b43b233, entries=150, sequenceid=548, filesize=12.0 K 2024-11-12T18:15:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0aafd3e99ddf42da80b19ee2187bd0a1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0aafd3e99ddf42da80b19ee2187bd0a1 2024-11-12T18:15:08,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0aafd3e99ddf42da80b19ee2187bd0a1, entries=150, sequenceid=548, filesize=12.0 K 2024-11-12T18:15:08,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 854ms, sequenceid=548, compaction requested=true 2024-11-12T18:15:08,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:08,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:08,539 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:08,539 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:08,539 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:08,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:08,540 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:08,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:08,540 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,540 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,540 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/16afd45b97d149d7a84eccea6cc0a585, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/15ac48727aa743f4870844adb60cd3ba, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/e775fe114c4e4f70b1def1209b43b233] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.9 K 2024-11-12T18:15:08,540 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb5f7c50e33144a2802ca46c7e52a24a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3f5cd3dfae564e83a61a1b2e486a0a30, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c5de1489a47a41fd92645a140905d855] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=39.3 K 2024-11-12T18:15:08,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 16afd45b97d149d7a84eccea6cc0a585, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731435305189 2024-11-12T18:15:08,540 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb5f7c50e33144a2802ca46c7e52a24a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731435305189 2024-11-12T18:15:08,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ac48727aa743f4870844adb60cd3ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731435305480 2024-11-12T18:15:08,541 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5cd3dfae564e83a61a1b2e486a0a30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731435305480 2024-11-12T18:15:08,541 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e775fe114c4e4f70b1def1209b43b233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731435306261 2024-11-12T18:15:08,541 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5de1489a47a41fd92645a140905d855, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731435306261 2024-11-12T18:15:08,550 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:08,550 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3c67be4b8f984550ae5aeb9bbf802288 is 50, key is test_row_0/A:col10/1731435306271/Put/seqid=0 2024-11-12T18:15:08,553 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:08,554 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9397476cd8744f52b8175a0a942fdc28 is 50, key is test_row_0/B:col10/1731435306271/Put/seqid=0 2024-11-12T18:15:08,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742156_1332 (size=13323) 2024-11-12T18:15:08,576 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/9397476cd8744f52b8175a0a942fdc28 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9397476cd8744f52b8175a0a942fdc28 2024-11-12T18:15:08,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742155_1331 (size=13323) 2024-11-12T18:15:08,583 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 9397476cd8744f52b8175a0a942fdc28(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:08,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:08,583 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435308539; duration=0sec 2024-11-12T18:15:08,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:08,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:08,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:08,584 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:08,584 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:08,584 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3c67be4b8f984550ae5aeb9bbf802288 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3c67be4b8f984550ae5aeb9bbf802288 2024-11-12T18:15:08,584 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,584 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/1fadc627e8ed41eb8982aee6a57e576f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/584e64b3b4894d03b9434f165cbc6c71, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0aafd3e99ddf42da80b19ee2187bd0a1] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=36.9 K 2024-11-12T18:15:08,585 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fadc627e8ed41eb8982aee6a57e576f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731435305189 2024-11-12T18:15:08,585 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 584e64b3b4894d03b9434f165cbc6c71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731435305480 2024-11-12T18:15:08,586 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aafd3e99ddf42da80b19ee2187bd0a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731435306261 2024-11-12T18:15:08,591 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 3c67be4b8f984550ae5aeb9bbf802288(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:08,591 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:08,591 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435308538; duration=0sec 2024-11-12T18:15:08,591 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:08,591 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:08,596 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#286 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:08,596 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0e68840adbb54e93a18603b54a8866f3 is 50, key is test_row_0/C:col10/1731435306271/Put/seqid=0 2024-11-12T18:15:08,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742157_1333 (size=13323) 2024-11-12T18:15:08,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T18:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:08,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:08,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/501783f082cb402ebdd7c6b2db540e29 is 50, key is test_row_0/A:col10/1731435307754/Put/seqid=0 2024-11-12T18:15:08,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742158_1334 (size=12301) 2024-11-12T18:15:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:08,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435368960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:09,009 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/0e68840adbb54e93a18603b54a8866f3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0e68840adbb54e93a18603b54a8866f3 2024-11-12T18:15:09,014 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 0e68840adbb54e93a18603b54a8866f3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:09,014 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:09,014 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435308539; duration=0sec 2024-11-12T18:15:09,015 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:09,015 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T18:15:09,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435369070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:09,090 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/501783f082cb402ebdd7c6b2db540e29 2024-11-12T18:15:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8b07d3b76fb74dc79e52f6f450408328 is 50, key is test_row_0/B:col10/1731435307754/Put/seqid=0 2024-11-12T18:15:09,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742159_1335 (size=12301) 2024-11-12T18:15:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435369281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:09,507 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8b07d3b76fb74dc79e52f6f450408328 2024-11-12T18:15:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5154aaf85b4a495a9813ae3ef22628bb is 50, key is test_row_0/C:col10/1731435307754/Put/seqid=0 2024-11-12T18:15:09,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742160_1336 (size=12301) 2024-11-12T18:15:09,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435369590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:09,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:09,920 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5154aaf85b4a495a9813ae3ef22628bb 2024-11-12T18:15:09,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/501783f082cb402ebdd7c6b2db540e29 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/501783f082cb402ebdd7c6b2db540e29 2024-11-12T18:15:09,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/501783f082cb402ebdd7c6b2db540e29, entries=150, sequenceid=570, filesize=12.0 K 2024-11-12T18:15:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8b07d3b76fb74dc79e52f6f450408328 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8b07d3b76fb74dc79e52f6f450408328 2024-11-12T18:15:09,933 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8b07d3b76fb74dc79e52f6f450408328, entries=150, sequenceid=570, filesize=12.0 K 2024-11-12T18:15:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/5154aaf85b4a495a9813ae3ef22628bb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5154aaf85b4a495a9813ae3ef22628bb 2024-11-12T18:15:09,938 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5154aaf85b4a495a9813ae3ef22628bb, entries=150, sequenceid=570, filesize=12.0 K 2024-11-12T18:15:09,939 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1267ms, sequenceid=570, compaction requested=false 2024-11-12T18:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-12T18:15:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-12T18:15:09,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T18:15:09,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-11-12T18:15:09,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.0410 sec 2024-11-12T18:15:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T18:15:10,051 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:10,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-12T18:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T18:15:10,053 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:10,054 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:10,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:10,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:15:10,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:10,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:10,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:10,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:10,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:10,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:10,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/6cdf217c67df4b4cb37edee44a6e19cc is 50, key is test_row_0/A:col10/1731435308942/Put/seqid=0 2024-11-12T18:15:10,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742161_1337 (size=14741) 2024-11-12T18:15:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T18:15:10,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435370178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:10,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:10,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435370290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:10,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T18:15:10,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59098 deadline: 1731435370450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:10,455 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59142 deadline: 1731435370460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:10,464 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:10,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435370500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:10,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/6cdf217c67df4b4cb37edee44a6e19cc 2024-11-12T18:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/1f0aa270f07249138262e3130361b0d4 is 50, key is test_row_0/B:col10/1731435308942/Put/seqid=0 2024-11-12T18:15:10,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742162_1338 (size=12301) 2024-11-12T18:15:10,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/1f0aa270f07249138262e3130361b0d4 2024-11-12T18:15:10,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/eafb5a2f2eda4f0695522b6e40a43b1f is 50, key is test_row_0/C:col10/1731435308942/Put/seqid=0 2024-11-12T18:15:10,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742163_1339 (size=12301) 2024-11-12T18:15:10,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T18:15:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435370810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:10,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:10,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:10,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/eafb5a2f2eda4f0695522b6e40a43b1f 2024-11-12T18:15:10,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/6cdf217c67df4b4cb37edee44a6e19cc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6cdf217c67df4b4cb37edee44a6e19cc 2024-11-12T18:15:10,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6cdf217c67df4b4cb37edee44a6e19cc, entries=200, sequenceid=584, filesize=14.4 K 2024-11-12T18:15:10,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/1f0aa270f07249138262e3130361b0d4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/1f0aa270f07249138262e3130361b0d4 2024-11-12T18:15:10,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:10,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:10,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:10,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/1f0aa270f07249138262e3130361b0d4, entries=150, sequenceid=584, filesize=12.0 K 2024-11-12T18:15:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/eafb5a2f2eda4f0695522b6e40a43b1f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/eafb5a2f2eda4f0695522b6e40a43b1f 2024-11-12T18:15:10,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/eafb5a2f2eda4f0695522b6e40a43b1f, entries=150, sequenceid=584, filesize=12.0 K 2024-11-12T18:15:10,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 879ms, sequenceid=584, compaction requested=true 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:10,983 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:10,983 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:10,984 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:10,984 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:10,984 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,985 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:10,985 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:10,985 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3c67be4b8f984550ae5aeb9bbf802288, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/501783f082cb402ebdd7c6b2db540e29, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6cdf217c67df4b4cb37edee44a6e19cc] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=39.4 K 2024-11-12T18:15:10,985 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:10,985 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9397476cd8744f52b8175a0a942fdc28, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8b07d3b76fb74dc79e52f6f450408328, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/1f0aa270f07249138262e3130361b0d4] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=37.0 K 2024-11-12T18:15:10,985 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c67be4b8f984550ae5aeb9bbf802288, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731435306261 2024-11-12T18:15:10,985 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9397476cd8744f52b8175a0a942fdc28, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731435306261 2024-11-12T18:15:10,985 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 501783f082cb402ebdd7c6b2db540e29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731435307754 2024-11-12T18:15:10,986 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b07d3b76fb74dc79e52f6f450408328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731435307754 2024-11-12T18:15:10,986 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdf217c67df4b4cb37edee44a6e19cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731435308942 2024-11-12T18:15:10,986 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0aa270f07249138262e3130361b0d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731435308942 2024-11-12T18:15:10,994 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:10,994 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/6587e31534524683961ad7ff0acb388b is 50, key is test_row_0/A:col10/1731435308942/Put/seqid=0 2024-11-12T18:15:11,006 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:11,006 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/4495a2a61a744a4daca24e512e9846b8 is 50, key is test_row_0/B:col10/1731435308942/Put/seqid=0 2024-11-12T18:15:11,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742164_1340 (size=13425) 2024-11-12T18:15:11,015 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/6587e31534524683961ad7ff0acb388b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6587e31534524683961ad7ff0acb388b 2024-11-12T18:15:11,021 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 6587e31534524683961ad7ff0acb388b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:11,021 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:11,021 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435310983; duration=0sec 2024-11-12T18:15:11,021 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:11,021 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:11,022 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:11,023 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:11,023 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:11,023 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:11,023 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0e68840adbb54e93a18603b54a8866f3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5154aaf85b4a495a9813ae3ef22628bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/eafb5a2f2eda4f0695522b6e40a43b1f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=37.0 K 2024-11-12T18:15:11,023 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e68840adbb54e93a18603b54a8866f3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731435306261 2024-11-12T18:15:11,024 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5154aaf85b4a495a9813ae3ef22628bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731435307754 2024-11-12T18:15:11,024 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafb5a2f2eda4f0695522b6e40a43b1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731435308942 2024-11-12T18:15:11,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742165_1341 (size=13425) 2024-11-12T18:15:11,049 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/4495a2a61a744a4daca24e512e9846b8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4495a2a61a744a4daca24e512e9846b8 2024-11-12T18:15:11,058 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#295 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:11,058 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/af01b9a7ad4a43ccb2aa74218e631c59 is 50, key is test_row_0/C:col10/1731435308942/Put/seqid=0 2024-11-12T18:15:11,060 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 4495a2a61a744a4daca24e512e9846b8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:11,060 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:11,060 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435310983; duration=0sec 2024-11-12T18:15:11,060 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:11,060 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:11,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742166_1342 (size=13425) 2024-11-12T18:15:11,077 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/af01b9a7ad4a43ccb2aa74218e631c59 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/af01b9a7ad4a43ccb2aa74218e631c59 2024-11-12T18:15:11,082 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into af01b9a7ad4a43ccb2aa74218e631c59(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:11,082 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:11,082 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435310983; duration=0sec 2024-11-12T18:15:11,083 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:11,083 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:11,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-12T18:15:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:11,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:15:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:11,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e4f8a538f67e426489203be7a90a8244 is 50, key is test_row_0/A:col10/1731435310171/Put/seqid=0 2024-11-12T18:15:11,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742167_1343 (size=12301) 2024-11-12T18:15:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T18:15:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:11,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:11,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435371361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435371470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:11,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e4f8a538f67e426489203be7a90a8244 2024-11-12T18:15:11,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8fc4eaec8ca74f838c35b02baa812cdf is 50, key is test_row_0/B:col10/1731435310171/Put/seqid=0 2024-11-12T18:15:11,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742168_1344 (size=12301) 2024-11-12T18:15:11,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8fc4eaec8ca74f838c35b02baa812cdf 2024-11-12T18:15:11,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/dad7727c6636445d86bebc5bb8d8516a is 50, key is test_row_0/C:col10/1731435310171/Put/seqid=0 2024-11-12T18:15:11,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742169_1345 (size=12301) 2024-11-12T18:15:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435371680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:11,965 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/dad7727c6636445d86bebc5bb8d8516a 2024-11-12T18:15:11,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e4f8a538f67e426489203be7a90a8244 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4f8a538f67e426489203be7a90a8244 2024-11-12T18:15:11,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4f8a538f67e426489203be7a90a8244, entries=150, sequenceid=606, filesize=12.0 K 2024-11-12T18:15:11,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/8fc4eaec8ca74f838c35b02baa812cdf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8fc4eaec8ca74f838c35b02baa812cdf 2024-11-12T18:15:11,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8fc4eaec8ca74f838c35b02baa812cdf, entries=150, sequenceid=606, filesize=12.0 K 2024-11-12T18:15:11,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/dad7727c6636445d86bebc5bb8d8516a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/dad7727c6636445d86bebc5bb8d8516a 2024-11-12T18:15:11,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/dad7727c6636445d86bebc5bb8d8516a, entries=150, sequenceid=606, filesize=12.0 K 2024-11-12T18:15:11,986 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 860ms, sequenceid=606, compaction requested=false 2024-11-12T18:15:11,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:11,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:11,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-12T18:15:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-12T18:15:11,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-12T18:15:11,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9330 sec 2024-11-12T18:15:11,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.9400 sec 2024-11-12T18:15:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:11,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:15:11,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:11,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:11,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:11,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:11,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:11,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:12,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/06ab98f4808a4c799172520cda6cb978 is 50, key is test_row_0/A:col10/1731435311359/Put/seqid=0 2024-11-12T18:15:12,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742170_1346 (size=12301) 2024-11-12T18:15:12,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435372067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:12,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435372180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T18:15:12,211 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:12,211 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-12T18:15:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T18:15:12,213 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:12,214 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:12,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T18:15:12,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:12,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:12,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435372390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:12,407 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-12T18:15:12,407 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:12,407 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,407 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:12,409 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-12T18:15:12,409 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:12,409 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,409 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:12,411 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-12T18:15:12,411 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:12,411 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,411 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:12,412 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-12T18:15:12,412 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:12,412 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,412 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:12,413 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-12T18:15:12,413 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:12,413 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,413 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:12,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:12,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/06ab98f4808a4c799172520cda6cb978 2024-11-12T18:15:12,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/10463c4543794f189cc3613afe01ac66 is 50, key is test_row_0/B:col10/1731435311359/Put/seqid=0 2024-11-12T18:15:12,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742171_1347 (size=12301) 2024-11-12T18:15:12,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59086 deadline: 1731435372490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:12,491 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:12,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T18:15:12,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1731435372550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:12,551 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_2' in a393ec6eeb0c6b3b0d3ff894a1f1a04b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:12,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435372700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:12,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/10463c4543794f189cc3613afe01ac66 2024-11-12T18:15:12,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:12,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:12,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/8d9179c8536d40b1b460f343a56b5fc4 is 50, key is test_row_0/C:col10/1731435311359/Put/seqid=0 2024-11-12T18:15:12,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742172_1348 (size=12301) 2024-11-12T18:15:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T18:15:12,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:12,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:12,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:13,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. as already flushing 2024-11-12T18:15:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:13,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:13,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59178 deadline: 1731435373210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:13,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/8d9179c8536d40b1b460f343a56b5fc4 2024-11-12T18:15:13,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/06ab98f4808a4c799172520cda6cb978 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06ab98f4808a4c799172520cda6cb978 2024-11-12T18:15:13,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06ab98f4808a4c799172520cda6cb978, entries=150, sequenceid=620, filesize=12.0 K 2024-11-12T18:15:13,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/10463c4543794f189cc3613afe01ac66 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/10463c4543794f189cc3613afe01ac66 2024-11-12T18:15:13,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/10463c4543794f189cc3613afe01ac66, entries=150, sequenceid=620, filesize=12.0 K 2024-11-12T18:15:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/8d9179c8536d40b1b460f343a56b5fc4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8d9179c8536d40b1b460f343a56b5fc4 2024-11-12T18:15:13,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8d9179c8536d40b1b460f343a56b5fc4, entries=150, sequenceid=620, filesize=12.0 K 2024-11-12T18:15:13,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 1264ms, sequenceid=620, compaction requested=true 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:13,260 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a393ec6eeb0c6b3b0d3ff894a1f1a04b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:13,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:15:13,260 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:13,261 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:13,261 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/A is initiating minor compaction (all files) 2024-11-12T18:15:13,261 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/A in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,261 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6587e31534524683961ad7ff0acb388b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4f8a538f67e426489203be7a90a8244, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06ab98f4808a4c799172520cda6cb978] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=37.1 K 2024-11-12T18:15:13,262 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:13,262 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6587e31534524683961ad7ff0acb388b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731435308942 2024-11-12T18:15:13,262 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/B is initiating minor compaction (all files) 2024-11-12T18:15:13,262 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/B in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,262 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4495a2a61a744a4daca24e512e9846b8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8fc4eaec8ca74f838c35b02baa812cdf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/10463c4543794f189cc3613afe01ac66] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=37.1 K 2024-11-12T18:15:13,262 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f8a538f67e426489203be7a90a8244, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731435310156 2024-11-12T18:15:13,262 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4495a2a61a744a4daca24e512e9846b8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731435308942 2024-11-12T18:15:13,263 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ab98f4808a4c799172520cda6cb978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1731435311333 2024-11-12T18:15:13,263 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc4eaec8ca74f838c35b02baa812cdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731435310156 2024-11-12T18:15:13,263 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 10463c4543794f189cc3613afe01ac66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1731435311333 2024-11-12T18:15:13,270 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#A#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:13,271 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/595a6642c1174e53996de06d0f61f7e2 is 50, key is test_row_0/A:col10/1731435311359/Put/seqid=0 2024-11-12T18:15:13,271 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:13,271 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/6a7d078512d44d78ae7892c55553380d is 50, key is test_row_0/B:col10/1731435311359/Put/seqid=0 2024-11-12T18:15:13,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742173_1349 (size=13527) 2024-11-12T18:15:13,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:13,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742174_1350 (size=13527) 2024-11-12T18:15:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e07d057a81114f7a9facb78f85728a62 is 50, key is test_row_0/A:col10/1731435312053/Put/seqid=0 2024-11-12T18:15:13,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742175_1351 (size=12301) 2024-11-12T18:15:13,308 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e07d057a81114f7a9facb78f85728a62 2024-11-12T18:15:13,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/b83437737a8949d8bcf8800f77d90e60 is 50, key is test_row_0/B:col10/1731435312053/Put/seqid=0 2024-11-12T18:15:13,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742176_1352 (size=12301) 2024-11-12T18:15:13,326 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/b83437737a8949d8bcf8800f77d90e60 2024-11-12T18:15:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c7917aa0c71a4d239d8b1449e958c40e is 50, key is test_row_0/C:col10/1731435312053/Put/seqid=0 2024-11-12T18:15:13,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742177_1353 (size=12301) 2024-11-12T18:15:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T18:15:13,690 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/595a6642c1174e53996de06d0f61f7e2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/595a6642c1174e53996de06d0f61f7e2 2024-11-12T18:15:13,697 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/A of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 595a6642c1174e53996de06d0f61f7e2(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:13,697 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:13,697 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, priority=13, startTime=1731435313260; duration=0sec 2024-11-12T18:15:13,697 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:13,697 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:A 2024-11-12T18:15:13,698 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:13,699 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:13,699 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): a393ec6eeb0c6b3b0d3ff894a1f1a04b/C is initiating minor compaction (all files) 2024-11-12T18:15:13,699 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/6a7d078512d44d78ae7892c55553380d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6a7d078512d44d78ae7892c55553380d 2024-11-12T18:15:13,699 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a393ec6eeb0c6b3b0d3ff894a1f1a04b/C in TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,700 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/af01b9a7ad4a43ccb2aa74218e631c59, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/dad7727c6636445d86bebc5bb8d8516a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8d9179c8536d40b1b460f343a56b5fc4] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp, totalSize=37.1 K 2024-11-12T18:15:13,700 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting af01b9a7ad4a43ccb2aa74218e631c59, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731435308942 2024-11-12T18:15:13,700 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad7727c6636445d86bebc5bb8d8516a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731435310156 2024-11-12T18:15:13,701 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9179c8536d40b1b460f343a56b5fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1731435311333 2024-11-12T18:15:13,705 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/B of a393ec6eeb0c6b3b0d3ff894a1f1a04b into 6a7d078512d44d78ae7892c55553380d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:13,705 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:13,705 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, priority=13, startTime=1731435313260; duration=0sec 2024-11-12T18:15:13,705 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:13,705 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:B 2024-11-12T18:15:13,711 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a393ec6eeb0c6b3b0d3ff894a1f1a04b#C#compaction#307 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:13,711 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ee82ce54ec6f4ba28c578417a84357e9 is 50, key is test_row_0/C:col10/1731435311359/Put/seqid=0 2024-11-12T18:15:13,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742178_1354 (size=13527) 2024-11-12T18:15:13,746 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c7917aa0c71a4d239d8b1449e958c40e 2024-11-12T18:15:13,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/e07d057a81114f7a9facb78f85728a62 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e07d057a81114f7a9facb78f85728a62 2024-11-12T18:15:13,758 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e07d057a81114f7a9facb78f85728a62, entries=150, sequenceid=638, filesize=12.0 K 2024-11-12T18:15:13,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/b83437737a8949d8bcf8800f77d90e60 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/b83437737a8949d8bcf8800f77d90e60 2024-11-12T18:15:13,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/b83437737a8949d8bcf8800f77d90e60, entries=150, sequenceid=638, filesize=12.0 K 2024-11-12T18:15:13,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/c7917aa0c71a4d239d8b1449e958c40e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c7917aa0c71a4d239d8b1449e958c40e 2024-11-12T18:15:13,781 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c7917aa0c71a4d239d8b1449e958c40e, entries=150, sequenceid=638, filesize=12.0 K 2024-11-12T18:15:13,782 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 497ms, sequenceid=638, compaction requested=false 2024-11-12T18:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-12T18:15:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-12T18:15:13,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-12T18:15:13,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-12T18:15:13,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.5770 sec 2024-11-12T18:15:14,127 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/ee82ce54ec6f4ba28c578417a84357e9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ee82ce54ec6f4ba28c578417a84357e9 2024-11-12T18:15:14,133 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a393ec6eeb0c6b3b0d3ff894a1f1a04b/C of a393ec6eeb0c6b3b0d3ff894a1f1a04b into ee82ce54ec6f4ba28c578417a84357e9(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:14,133 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: 2024-11-12T18:15:14,133 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., storeName=a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, priority=13, startTime=1731435313260; duration=0sec 2024-11-12T18:15:14,133 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:14,133 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a393ec6eeb0c6b3b0d3ff894a1f1a04b:C 2024-11-12T18:15:14,222 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-12T18:15:14,222 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:14,222 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:14,222 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:14,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-12T18:15:14,361 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:14,492 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-12T18:15:14,492 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:14,492 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:14,492 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:14,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:14,501 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-12T18:15:14,501 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:14,501 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:14,501 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:14,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:21,242 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T18:15:22,522 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-12T18:15:22,522 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:22,523 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:22,523 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:22,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:22,642 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-12T18:15:22,642 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:22,642 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:22,642 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:22,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:22,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 281 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2535 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2520 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2415 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2543 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2522 2024-11-12T18:15:22,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T18:15:22,643 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:15:22,643 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:15:22,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:22,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:22,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:22,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T18:15:22,647 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435322647"}]},"ts":"1731435322647"} 2024-11-12T18:15:22,648 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T18:15:22,648 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T18:15:22,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:15:22,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, UNASSIGN}] 2024-11-12T18:15:22,651 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, UNASSIGN 2024-11-12T18:15:22,652 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a393ec6eeb0c6b3b0d3ff894a1f1a04b, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:22,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, UNASSIGN because future has completed 2024-11-12T18:15:22,656 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:15:22,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:15:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T18:15:22,809 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing a393ec6eeb0c6b3b0d3ff894a1f1a04b, disabling compactions & flushes 2024-11-12T18:15:22,809 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. after waiting 0 ms 2024-11-12T18:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:22,809 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T18:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=A 2024-11-12T18:15:22,810 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:22,810 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=B 2024-11-12T18:15:22,810 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:22,810 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a393ec6eeb0c6b3b0d3ff894a1f1a04b, store=C 2024-11-12T18:15:22,810 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:22,813 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3a43cb09db724ef49235d3a2f102d02a is 50, key is test_row_0/A:col10/1731435322521/Put/seqid=0 2024-11-12T18:15:22,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742179_1355 (size=12301) 2024-11-12T18:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T18:15:23,218 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3a43cb09db724ef49235d3a2f102d02a 2024-11-12T18:15:23,230 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/bd5779de359f451a851627c478ab4db4 is 50, key is test_row_0/B:col10/1731435322521/Put/seqid=0 2024-11-12T18:15:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742180_1356 (size=12301) 2024-11-12T18:15:23,245 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/bd5779de359f451a851627c478ab4db4 2024-11-12T18:15:23,262 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/d78f6ec317024d10b22039905a2e46a8 is 50, key is test_row_0/C:col10/1731435322521/Put/seqid=0 2024-11-12T18:15:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742181_1357 (size=12301) 2024-11-12T18:15:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T18:15:23,270 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/d78f6ec317024d10b22039905a2e46a8 2024-11-12T18:15:23,275 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/A/3a43cb09db724ef49235d3a2f102d02a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3a43cb09db724ef49235d3a2f102d02a 2024-11-12T18:15:23,282 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3a43cb09db724ef49235d3a2f102d02a, entries=150, sequenceid=649, filesize=12.0 K 2024-11-12T18:15:23,283 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/B/bd5779de359f451a851627c478ab4db4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bd5779de359f451a851627c478ab4db4 2024-11-12T18:15:23,296 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bd5779de359f451a851627c478ab4db4, entries=150, sequenceid=649, filesize=12.0 K 2024-11-12T18:15:23,297 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/.tmp/C/d78f6ec317024d10b22039905a2e46a8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d78f6ec317024d10b22039905a2e46a8 2024-11-12T18:15:23,304 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d78f6ec317024d10b22039905a2e46a8, entries=150, sequenceid=649, filesize=12.0 K 2024-11-12T18:15:23,305 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 496ms, sequenceid=649, compaction requested=true 2024-11-12T18:15:23,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8d6d750e96924bf29e1a116c85d805b1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/56932395b2cb43e0b1f8a31796074496, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9565c083e9504fe7bcdfe18c9be9eaee, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb9b37c337e54141b7d6cd17d167d248, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/d741b48d91de49278e58ef7858fc5372, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/03bb15c0dadc43adbff5e58f889b3459, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/674706f137f84165859f7c31940af430, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/b245b8c912664efea93e0ffb2d9f1e41, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8eb1fde002054ee58b833e2c5b96030f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8797d2458094405f8a4cf705c6de5423, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ea8f608b6003449eb7d4f7d16ba4ffd6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3cc7b95394be4b1594453cc3296a7ba8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/692b027b9afb4ee99e6060da43a1f05f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06f8ba07f3a94d318833cd8769f1cfa8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8282d01d6e904202b4786f10e2a71693, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/4f1283f37c6f44c7a6af5c93c1ec3473, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a8a9b1a2c22046a2b1683c5aab1f7db8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c6caea84f34f44ecb2dfdc4449164580, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a06f9bc52c0947d98522eed7742873c0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/02f5ac1f602a4993ac454109e00dbdaf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e06cce1857d544268f5afdcec41b04c0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4ebb80f52924d8895551a63da9e8748, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/db7bfeff7fe848b98babe8fcdee11722, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c4aff2c167bd4898ba8d1b514fcc1ab7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9715896d325a48d99dd01bdcf8f9f3f6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ae07ab7269954e1ba2a18ccdb4414c0d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb5f7c50e33144a2802ca46c7e52a24a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/61172cc07daf4818b41e77883c72ad17, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3f5cd3dfae564e83a61a1b2e486a0a30, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c5de1489a47a41fd92645a140905d855, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3c67be4b8f984550ae5aeb9bbf802288, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/501783f082cb402ebdd7c6b2db540e29, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6cdf217c67df4b4cb37edee44a6e19cc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6587e31534524683961ad7ff0acb388b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4f8a538f67e426489203be7a90a8244, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06ab98f4808a4c799172520cda6cb978] to archive 2024-11-12T18:15:23,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:15:23,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8d6d750e96924bf29e1a116c85d805b1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8d6d750e96924bf29e1a116c85d805b1 2024-11-12T18:15:23,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/56932395b2cb43e0b1f8a31796074496 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/56932395b2cb43e0b1f8a31796074496 2024-11-12T18:15:23,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9565c083e9504fe7bcdfe18c9be9eaee to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9565c083e9504fe7bcdfe18c9be9eaee 2024-11-12T18:15:23,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb9b37c337e54141b7d6cd17d167d248 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb9b37c337e54141b7d6cd17d167d248 2024-11-12T18:15:23,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/d741b48d91de49278e58ef7858fc5372 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/d741b48d91de49278e58ef7858fc5372 2024-11-12T18:15:23,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/03bb15c0dadc43adbff5e58f889b3459 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/03bb15c0dadc43adbff5e58f889b3459 2024-11-12T18:15:23,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/674706f137f84165859f7c31940af430 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/674706f137f84165859f7c31940af430 2024-11-12T18:15:23,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/b245b8c912664efea93e0ffb2d9f1e41 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/b245b8c912664efea93e0ffb2d9f1e41 2024-11-12T18:15:23,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8eb1fde002054ee58b833e2c5b96030f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8eb1fde002054ee58b833e2c5b96030f 2024-11-12T18:15:23,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8797d2458094405f8a4cf705c6de5423 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8797d2458094405f8a4cf705c6de5423 2024-11-12T18:15:23,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ea8f608b6003449eb7d4f7d16ba4ffd6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ea8f608b6003449eb7d4f7d16ba4ffd6 2024-11-12T18:15:23,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3cc7b95394be4b1594453cc3296a7ba8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3cc7b95394be4b1594453cc3296a7ba8 2024-11-12T18:15:23,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/692b027b9afb4ee99e6060da43a1f05f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/692b027b9afb4ee99e6060da43a1f05f 2024-11-12T18:15:23,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06f8ba07f3a94d318833cd8769f1cfa8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06f8ba07f3a94d318833cd8769f1cfa8 2024-11-12T18:15:23,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8282d01d6e904202b4786f10e2a71693 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/8282d01d6e904202b4786f10e2a71693 2024-11-12T18:15:23,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/4f1283f37c6f44c7a6af5c93c1ec3473 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/4f1283f37c6f44c7a6af5c93c1ec3473 2024-11-12T18:15:23,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a8a9b1a2c22046a2b1683c5aab1f7db8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a8a9b1a2c22046a2b1683c5aab1f7db8 2024-11-12T18:15:23,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c6caea84f34f44ecb2dfdc4449164580 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c6caea84f34f44ecb2dfdc4449164580 2024-11-12T18:15:23,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a06f9bc52c0947d98522eed7742873c0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/a06f9bc52c0947d98522eed7742873c0 2024-11-12T18:15:23,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/02f5ac1f602a4993ac454109e00dbdaf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/02f5ac1f602a4993ac454109e00dbdaf 2024-11-12T18:15:23,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e06cce1857d544268f5afdcec41b04c0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e06cce1857d544268f5afdcec41b04c0 2024-11-12T18:15:23,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4ebb80f52924d8895551a63da9e8748 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4ebb80f52924d8895551a63da9e8748 2024-11-12T18:15:23,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/db7bfeff7fe848b98babe8fcdee11722 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/db7bfeff7fe848b98babe8fcdee11722 2024-11-12T18:15:23,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c4aff2c167bd4898ba8d1b514fcc1ab7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c4aff2c167bd4898ba8d1b514fcc1ab7 2024-11-12T18:15:23,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9715896d325a48d99dd01bdcf8f9f3f6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/9715896d325a48d99dd01bdcf8f9f3f6 2024-11-12T18:15:23,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ae07ab7269954e1ba2a18ccdb4414c0d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/ae07ab7269954e1ba2a18ccdb4414c0d 2024-11-12T18:15:23,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb5f7c50e33144a2802ca46c7e52a24a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/eb5f7c50e33144a2802ca46c7e52a24a 2024-11-12T18:15:23,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/61172cc07daf4818b41e77883c72ad17 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/61172cc07daf4818b41e77883c72ad17 2024-11-12T18:15:23,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3f5cd3dfae564e83a61a1b2e486a0a30 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3f5cd3dfae564e83a61a1b2e486a0a30 2024-11-12T18:15:23,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c5de1489a47a41fd92645a140905d855 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/c5de1489a47a41fd92645a140905d855 2024-11-12T18:15:23,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3c67be4b8f984550ae5aeb9bbf802288 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3c67be4b8f984550ae5aeb9bbf802288 2024-11-12T18:15:23,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/501783f082cb402ebdd7c6b2db540e29 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/501783f082cb402ebdd7c6b2db540e29 2024-11-12T18:15:23,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6cdf217c67df4b4cb37edee44a6e19cc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6cdf217c67df4b4cb37edee44a6e19cc 2024-11-12T18:15:23,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6587e31534524683961ad7ff0acb388b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/6587e31534524683961ad7ff0acb388b 2024-11-12T18:15:23,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4f8a538f67e426489203be7a90a8244 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e4f8a538f67e426489203be7a90a8244 2024-11-12T18:15:23,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06ab98f4808a4c799172520cda6cb978 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/06ab98f4808a4c799172520cda6cb978 2024-11-12T18:15:23,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/0023264875654149ba7157a706a4346b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d96c5421238748289d8bc24656fece07, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bf1675c129fa4623bed8f6bad42a694a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/af40ca50a65d42c78546bac3e07a1792, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d5a5d5354b714c1e8f39d8e27d62f7a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d6164778c0994101afaa000503d5ec46, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4137c8ef34074cfeb26feda4200823c6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7a4a15375e25484391f1df298e96eddf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9ca1c777527f48988dacfd060cb7c4eb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7c6b955bdd9d4286a1dbc57b193bdc0e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/21bb1f97bf2a4d1cbc1668282f049faa, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/fe8e06f2203f4af08c56bf10e064c3ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/eccdc6cdaf034d9e95dd673ae3fc0c5c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/2c927d8b7f7d45159649fb39e15dd655, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6b438c05bfd44fb396fd362884508b2b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9a36972a46c9457a8a46936d6ff4f52a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/07a419ee28314992b59c8809ab3c4ae5, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5bdef56108464be294d649cc9dbd0c45, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a30dacc3743749b6825c9a17984ddd2c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8a290b86671749269e7d2e9e3f952b00, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/40d7c2838f4d4a5eb64f7481e8b04314, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/71841e4fb65d4b5cb963d61c541296a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/3e04529b3c1348e2973c8bb8ecfbcf7a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/116eb8540cd64161b6fbfd07e585064b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5e1de8a5ed3d401c8cfa7622170e4227, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a8e0904ca70840c8bd0b6d093d2bb8a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/16afd45b97d149d7a84eccea6cc0a585, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/dbe0d499bd124181b3467c47ee792bf4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/15ac48727aa743f4870844adb60cd3ba, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9397476cd8744f52b8175a0a942fdc28, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/e775fe114c4e4f70b1def1209b43b233, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8b07d3b76fb74dc79e52f6f450408328, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4495a2a61a744a4daca24e512e9846b8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/1f0aa270f07249138262e3130361b0d4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8fc4eaec8ca74f838c35b02baa812cdf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/10463c4543794f189cc3613afe01ac66] to archive 2024-11-12T18:15:23,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:15:23,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/0023264875654149ba7157a706a4346b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/0023264875654149ba7157a706a4346b 2024-11-12T18:15:23,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d96c5421238748289d8bc24656fece07 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d96c5421238748289d8bc24656fece07 2024-11-12T18:15:23,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bf1675c129fa4623bed8f6bad42a694a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bf1675c129fa4623bed8f6bad42a694a 2024-11-12T18:15:23,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/af40ca50a65d42c78546bac3e07a1792 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/af40ca50a65d42c78546bac3e07a1792 2024-11-12T18:15:23,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d5a5d5354b714c1e8f39d8e27d62f7a4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d5a5d5354b714c1e8f39d8e27d62f7a4 2024-11-12T18:15:23,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d6164778c0994101afaa000503d5ec46 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/d6164778c0994101afaa000503d5ec46 2024-11-12T18:15:23,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4137c8ef34074cfeb26feda4200823c6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4137c8ef34074cfeb26feda4200823c6 2024-11-12T18:15:23,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7a4a15375e25484391f1df298e96eddf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7a4a15375e25484391f1df298e96eddf 2024-11-12T18:15:23,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9ca1c777527f48988dacfd060cb7c4eb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9ca1c777527f48988dacfd060cb7c4eb 2024-11-12T18:15:23,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7c6b955bdd9d4286a1dbc57b193bdc0e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/7c6b955bdd9d4286a1dbc57b193bdc0e 2024-11-12T18:15:23,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/21bb1f97bf2a4d1cbc1668282f049faa to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/21bb1f97bf2a4d1cbc1668282f049faa 2024-11-12T18:15:23,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/fe8e06f2203f4af08c56bf10e064c3ad to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/fe8e06f2203f4af08c56bf10e064c3ad 2024-11-12T18:15:23,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/eccdc6cdaf034d9e95dd673ae3fc0c5c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/eccdc6cdaf034d9e95dd673ae3fc0c5c 2024-11-12T18:15:23,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/2c927d8b7f7d45159649fb39e15dd655 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/2c927d8b7f7d45159649fb39e15dd655 2024-11-12T18:15:23,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6b438c05bfd44fb396fd362884508b2b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6b438c05bfd44fb396fd362884508b2b 2024-11-12T18:15:23,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9a36972a46c9457a8a46936d6ff4f52a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9a36972a46c9457a8a46936d6ff4f52a 2024-11-12T18:15:23,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/07a419ee28314992b59c8809ab3c4ae5 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/07a419ee28314992b59c8809ab3c4ae5 2024-11-12T18:15:23,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5bdef56108464be294d649cc9dbd0c45 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5bdef56108464be294d649cc9dbd0c45 2024-11-12T18:15:23,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a30dacc3743749b6825c9a17984ddd2c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a30dacc3743749b6825c9a17984ddd2c 2024-11-12T18:15:23,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8a290b86671749269e7d2e9e3f952b00 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8a290b86671749269e7d2e9e3f952b00 2024-11-12T18:15:23,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/40d7c2838f4d4a5eb64f7481e8b04314 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/40d7c2838f4d4a5eb64f7481e8b04314 2024-11-12T18:15:23,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/71841e4fb65d4b5cb963d61c541296a2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/71841e4fb65d4b5cb963d61c541296a2 2024-11-12T18:15:23,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/3e04529b3c1348e2973c8bb8ecfbcf7a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/3e04529b3c1348e2973c8bb8ecfbcf7a 2024-11-12T18:15:23,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/116eb8540cd64161b6fbfd07e585064b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/116eb8540cd64161b6fbfd07e585064b 2024-11-12T18:15:23,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5e1de8a5ed3d401c8cfa7622170e4227 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/5e1de8a5ed3d401c8cfa7622170e4227 2024-11-12T18:15:23,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a8e0904ca70840c8bd0b6d093d2bb8a4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/a8e0904ca70840c8bd0b6d093d2bb8a4 2024-11-12T18:15:23,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/16afd45b97d149d7a84eccea6cc0a585 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/16afd45b97d149d7a84eccea6cc0a585 2024-11-12T18:15:23,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/dbe0d499bd124181b3467c47ee792bf4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/dbe0d499bd124181b3467c47ee792bf4 2024-11-12T18:15:23,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/15ac48727aa743f4870844adb60cd3ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/15ac48727aa743f4870844adb60cd3ba 2024-11-12T18:15:23,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9397476cd8744f52b8175a0a942fdc28 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/9397476cd8744f52b8175a0a942fdc28 2024-11-12T18:15:23,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/e775fe114c4e4f70b1def1209b43b233 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/e775fe114c4e4f70b1def1209b43b233 2024-11-12T18:15:23,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8b07d3b76fb74dc79e52f6f450408328 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8b07d3b76fb74dc79e52f6f450408328 2024-11-12T18:15:23,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4495a2a61a744a4daca24e512e9846b8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/4495a2a61a744a4daca24e512e9846b8 2024-11-12T18:15:23,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/1f0aa270f07249138262e3130361b0d4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/1f0aa270f07249138262e3130361b0d4 2024-11-12T18:15:23,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8fc4eaec8ca74f838c35b02baa812cdf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/8fc4eaec8ca74f838c35b02baa812cdf 2024-11-12T18:15:23,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/10463c4543794f189cc3613afe01ac66 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/10463c4543794f189cc3613afe01ac66 2024-11-12T18:15:23,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/4ff2be856a6a4f9c84b79089f5256de1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ecf0bd2ec7f7426dbff92007566968e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/249de40a941b41238040c1fcdba33972, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/a41af412fdfb46e2bdf37e6c4110a3b0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ba35be7bbb9643d7bf5cc9247ea1cb8c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5881c1b14d54400e9fe3afa0541377dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/94c4ab84489e4f538916df0a958aaf6c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/23ff15ff18fb40ea8037bcddf524faa4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7618138ed0e64b579f7ed0d097944ced, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7bc757dd045549c49b5a31a2b1ce3ee5, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0414d7b16bf84c6f9678d714106771fd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/f7653e93aa194ad99abdc9b23eab39ee, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/415be9eefe2a4084bfd2ac1db57600d2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d18775c2211646a6837c62b5dec3b03c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/6931293504fc4d95be6eb0ecb9139ce2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c96f3b66063b4ee1b297ffebb4ac667a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0768a44d0fb94b86a60b6cdbd678c75b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0f86fb8d46f943ec9f5b40ab1df051ee, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/e930fdde47414a55ab8f469685095b74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/40e273a7266045c48e2bacfe4438a253, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/64cc578a94d443f78b31a98941945764, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c2067569ac6841e7bb7b0596ecd00b76, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5ed12be769f4471e81748c4f54ca7365, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/21de62ef3df64059acb0e7b591f065af, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8a0cdf3c6da6400a956110c371db6636, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0803e61f4d0649c3bbcfa55cba9d9158, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/1fadc627e8ed41eb8982aee6a57e576f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c84bff579f7348398f2179a266cf4ec9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/584e64b3b4894d03b9434f165cbc6c71, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0e68840adbb54e93a18603b54a8866f3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0aafd3e99ddf42da80b19ee2187bd0a1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5154aaf85b4a495a9813ae3ef22628bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/af01b9a7ad4a43ccb2aa74218e631c59, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/eafb5a2f2eda4f0695522b6e40a43b1f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/dad7727c6636445d86bebc5bb8d8516a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8d9179c8536d40b1b460f343a56b5fc4] to archive 2024-11-12T18:15:23,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:15:23,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/4ff2be856a6a4f9c84b79089f5256de1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/4ff2be856a6a4f9c84b79089f5256de1 2024-11-12T18:15:23,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ecf0bd2ec7f7426dbff92007566968e7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ecf0bd2ec7f7426dbff92007566968e7 2024-11-12T18:15:23,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/249de40a941b41238040c1fcdba33972 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/249de40a941b41238040c1fcdba33972 2024-11-12T18:15:23,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/a41af412fdfb46e2bdf37e6c4110a3b0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/a41af412fdfb46e2bdf37e6c4110a3b0 2024-11-12T18:15:23,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ba35be7bbb9643d7bf5cc9247ea1cb8c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ba35be7bbb9643d7bf5cc9247ea1cb8c 2024-11-12T18:15:23,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5881c1b14d54400e9fe3afa0541377dc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5881c1b14d54400e9fe3afa0541377dc 2024-11-12T18:15:23,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/94c4ab84489e4f538916df0a958aaf6c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/94c4ab84489e4f538916df0a958aaf6c 2024-11-12T18:15:23,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/23ff15ff18fb40ea8037bcddf524faa4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/23ff15ff18fb40ea8037bcddf524faa4 2024-11-12T18:15:23,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7618138ed0e64b579f7ed0d097944ced to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7618138ed0e64b579f7ed0d097944ced 2024-11-12T18:15:23,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7bc757dd045549c49b5a31a2b1ce3ee5 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/7bc757dd045549c49b5a31a2b1ce3ee5 2024-11-12T18:15:23,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0414d7b16bf84c6f9678d714106771fd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0414d7b16bf84c6f9678d714106771fd 2024-11-12T18:15:23,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/f7653e93aa194ad99abdc9b23eab39ee to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/f7653e93aa194ad99abdc9b23eab39ee 2024-11-12T18:15:23,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/415be9eefe2a4084bfd2ac1db57600d2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/415be9eefe2a4084bfd2ac1db57600d2 2024-11-12T18:15:23,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d18775c2211646a6837c62b5dec3b03c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d18775c2211646a6837c62b5dec3b03c 2024-11-12T18:15:23,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/6931293504fc4d95be6eb0ecb9139ce2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/6931293504fc4d95be6eb0ecb9139ce2 2024-11-12T18:15:23,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c96f3b66063b4ee1b297ffebb4ac667a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c96f3b66063b4ee1b297ffebb4ac667a 2024-11-12T18:15:23,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0768a44d0fb94b86a60b6cdbd678c75b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0768a44d0fb94b86a60b6cdbd678c75b 2024-11-12T18:15:23,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0f86fb8d46f943ec9f5b40ab1df051ee to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0f86fb8d46f943ec9f5b40ab1df051ee 2024-11-12T18:15:23,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/e930fdde47414a55ab8f469685095b74 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/e930fdde47414a55ab8f469685095b74 2024-11-12T18:15:23,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/40e273a7266045c48e2bacfe4438a253 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/40e273a7266045c48e2bacfe4438a253 2024-11-12T18:15:23,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/64cc578a94d443f78b31a98941945764 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/64cc578a94d443f78b31a98941945764 2024-11-12T18:15:23,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c2067569ac6841e7bb7b0596ecd00b76 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c2067569ac6841e7bb7b0596ecd00b76 2024-11-12T18:15:23,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5ed12be769f4471e81748c4f54ca7365 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5ed12be769f4471e81748c4f54ca7365 2024-11-12T18:15:23,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/21de62ef3df64059acb0e7b591f065af to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/21de62ef3df64059acb0e7b591f065af 2024-11-12T18:15:23,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8a0cdf3c6da6400a956110c371db6636 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8a0cdf3c6da6400a956110c371db6636 2024-11-12T18:15:23,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0803e61f4d0649c3bbcfa55cba9d9158 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0803e61f4d0649c3bbcfa55cba9d9158 2024-11-12T18:15:23,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/1fadc627e8ed41eb8982aee6a57e576f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/1fadc627e8ed41eb8982aee6a57e576f 2024-11-12T18:15:23,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c84bff579f7348398f2179a266cf4ec9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c84bff579f7348398f2179a266cf4ec9 2024-11-12T18:15:23,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/584e64b3b4894d03b9434f165cbc6c71 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/584e64b3b4894d03b9434f165cbc6c71 2024-11-12T18:15:23,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0e68840adbb54e93a18603b54a8866f3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0e68840adbb54e93a18603b54a8866f3 2024-11-12T18:15:23,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0aafd3e99ddf42da80b19ee2187bd0a1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/0aafd3e99ddf42da80b19ee2187bd0a1 2024-11-12T18:15:23,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5154aaf85b4a495a9813ae3ef22628bb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/5154aaf85b4a495a9813ae3ef22628bb 2024-11-12T18:15:23,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/af01b9a7ad4a43ccb2aa74218e631c59 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/af01b9a7ad4a43ccb2aa74218e631c59 2024-11-12T18:15:23,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/eafb5a2f2eda4f0695522b6e40a43b1f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/eafb5a2f2eda4f0695522b6e40a43b1f 2024-11-12T18:15:23,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/dad7727c6636445d86bebc5bb8d8516a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/dad7727c6636445d86bebc5bb8d8516a 2024-11-12T18:15:23,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8d9179c8536d40b1b460f343a56b5fc4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/8d9179c8536d40b1b460f343a56b5fc4 2024-11-12T18:15:23,482 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/recovered.edits/652.seqid, newMaxSeqId=652, maxSeqId=1 2024-11-12T18:15:23,483 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. 2024-11-12T18:15:23,483 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for a393ec6eeb0c6b3b0d3ff894a1f1a04b: Waiting for close lock at 1731435322809Running coprocessor pre-close hooks at 1731435322809Disabling compacts and flushes for region at 1731435322809Disabling writes for close at 1731435322809Obtaining lock to block concurrent updates at 1731435322809Preparing flush snapshotting stores in a393ec6eeb0c6b3b0d3ff894a1f1a04b at 1731435322809Finished memstore snapshotting TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731435322810 (+1 ms)Flushing stores of TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b. at 1731435322810Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/A: creating writer at 1731435322810Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/A: appending metadata at 1731435322813 (+3 ms)Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/A: closing flushed file at 1731435322813Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/B: creating writer at 1731435323222 (+409 ms)Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/B: appending metadata at 1731435323229 (+7 ms)Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/B: closing flushed file at 1731435323229Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/C: creating writer at 1731435323255 (+26 ms)Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/C: appending metadata at 1731435323261 (+6 ms)Flushing a393ec6eeb0c6b3b0d3ff894a1f1a04b/C: closing flushed file at 1731435323261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15894cef: reopening flushed file at 1731435323274 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d0f7b8a: reopening flushed file at 1731435323282 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d47b057: reopening flushed file at 1731435323296 (+14 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a393ec6eeb0c6b3b0d3ff894a1f1a04b in 496ms, sequenceid=649, compaction requested=true at 1731435323305 (+9 ms)Writing region close event to WAL at 1731435323478 (+173 ms)Running coprocessor post-close hooks at 1731435323482 (+4 ms)Closed at 1731435323482 2024-11-12T18:15:23,485 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:23,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a393ec6eeb0c6b3b0d3ff894a1f1a04b, regionState=CLOSED 2024-11-12T18:15:23,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:15:23,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-12T18:15:23,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure a393ec6eeb0c6b3b0d3ff894a1f1a04b, server=9911683f163c,43757,1731435233300 in 832 msec 2024-11-12T18:15:23,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-12T18:15:23,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a393ec6eeb0c6b3b0d3ff894a1f1a04b, UNASSIGN in 840 msec 2024-11-12T18:15:23,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-12T18:15:23,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 843 msec 2024-11-12T18:15:23,495 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435323494"}]},"ts":"1731435323494"} 2024-11-12T18:15:23,496 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T18:15:23,496 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T18:15:23,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 853 msec 2024-11-12T18:15:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T18:15:23,781 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:23,781 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T18:15:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,784 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-12T18:15:23,784 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:23,789 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/recovered.edits] 2024-11-12T18:15:23,792 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3a43cb09db724ef49235d3a2f102d02a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/3a43cb09db724ef49235d3a2f102d02a 2024-11-12T18:15:23,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/595a6642c1174e53996de06d0f61f7e2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/595a6642c1174e53996de06d0f61f7e2 2024-11-12T18:15:23,795 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e07d057a81114f7a9facb78f85728a62 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/A/e07d057a81114f7a9facb78f85728a62 2024-11-12T18:15:23,799 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6a7d078512d44d78ae7892c55553380d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/6a7d078512d44d78ae7892c55553380d 2024-11-12T18:15:23,801 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/b83437737a8949d8bcf8800f77d90e60 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/b83437737a8949d8bcf8800f77d90e60 2024-11-12T18:15:23,802 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bd5779de359f451a851627c478ab4db4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/B/bd5779de359f451a851627c478ab4db4 2024-11-12T18:15:23,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c7917aa0c71a4d239d8b1449e958c40e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/c7917aa0c71a4d239d8b1449e958c40e 2024-11-12T18:15:23,805 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d78f6ec317024d10b22039905a2e46a8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/d78f6ec317024d10b22039905a2e46a8 2024-11-12T18:15:23,806 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ee82ce54ec6f4ba28c578417a84357e9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/C/ee82ce54ec6f4ba28c578417a84357e9 2024-11-12T18:15:23,809 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/recovered.edits/652.seqid to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b/recovered.edits/652.seqid 2024-11-12T18:15:23,809 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/a393ec6eeb0c6b3b0d3ff894a1f1a04b 2024-11-12T18:15:23,809 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T18:15:23,811 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,815 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T18:15:23,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T18:15:23,819 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,819 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T18:15:23,819 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731435323819"}]},"ts":"9223372036854775807"} 2024-11-12T18:15:23,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T18:15:23,821 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a393ec6eeb0c6b3b0d3ff894a1f1a04b, NAME => 'TestAcidGuarantees,,1731435291151.a393ec6eeb0c6b3b0d3ff894a1f1a04b.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T18:15:23,821 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T18:15:23,822 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731435323821"}]},"ts":"9223372036854775807"} 2024-11-12T18:15:23,823 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T18:15:23,824 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-12T18:15:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-12T18:15:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T18:15:23,891 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:23,907 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=654 (was 759), ProcessCount=11 (was 11), AvailableMemoryMB=2130 (was 2059) - AvailableMemoryMB LEAK? - 2024-11-12T18:15:23,924 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=654, ProcessCount=11, AvailableMemoryMB=2128 2024-11-12T18:15:23,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:15:23,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:23,931 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:15:23,931 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:23,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-12T18:15:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T18:15:23,933 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:15:23,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742182_1358 (size=1061) 2024-11-12T18:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T18:15:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T18:15:24,367 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:15:24,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742183_1359 (size=53) 2024-11-12T18:15:24,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:15:24,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3946528656bca5567e369f3af446cf65, disabling compactions & flushes 2024-11-12T18:15:24,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. after waiting 0 ms 2024-11-12T18:15:24,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3946528656bca5567e369f3af446cf65: Waiting for close lock at 1731435324393Disabling compacts and flushes for region at 1731435324393Disabling writes for close at 1731435324393Writing region close event to WAL at 1731435324393Closed at 1731435324393 2024-11-12T18:15:24,395 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:15:24,395 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731435324395"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435324395"}]},"ts":"1731435324395"} 2024-11-12T18:15:24,398 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T18:15:24,399 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:15:24,399 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435324399"}]},"ts":"1731435324399"} 2024-11-12T18:15:24,402 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T18:15:24,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, ASSIGN}] 2024-11-12T18:15:24,404 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, ASSIGN 2024-11-12T18:15:24,405 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, ASSIGN; state=OFFLINE, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=false 2024-11-12T18:15:24,555 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:24,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, ASSIGN because future has completed 2024-11-12T18:15:24,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:15:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T18:15:24,712 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,712 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:15:24,713 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,713 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:15:24,713 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,713 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,714 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,715 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:24,715 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3946528656bca5567e369f3af446cf65 columnFamilyName A 2024-11-12T18:15:24,715 DEBUG [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:24,716 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(327): Store=3946528656bca5567e369f3af446cf65/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:24,716 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,717 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:24,717 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3946528656bca5567e369f3af446cf65 columnFamilyName B 2024-11-12T18:15:24,717 DEBUG [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:24,717 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(327): Store=3946528656bca5567e369f3af446cf65/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:24,717 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,718 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:24,719 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3946528656bca5567e369f3af446cf65 columnFamilyName C 2024-11-12T18:15:24,719 DEBUG [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:24,719 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(327): Store=3946528656bca5567e369f3af446cf65/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:24,719 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,719 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,720 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,720 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,721 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,721 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,721 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:15:24,722 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,724 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:15:24,725 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 3946528656bca5567e369f3af446cf65; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67204782, jitterRate=0.0014292895793914795}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:15:24,725 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:24,726 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 3946528656bca5567e369f3af446cf65: Running coprocessor pre-open hook at 1731435324713Writing region info on filesystem at 1731435324713Initializing all the Stores at 1731435324713Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435324713Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435324714 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435324714Cleaning up temporary data from old regions at 1731435324721 (+7 ms)Running coprocessor post-open hooks at 1731435324725 (+4 ms)Region opened successfully at 1731435324725 2024-11-12T18:15:24,726 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., pid=91, masterSystemTime=1731435324709 2024-11-12T18:15:24,728 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,728 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:24,729 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:24,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:15:24,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-12T18:15:24,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 in 174 msec 2024-11-12T18:15:24,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-12T18:15:24,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, ASSIGN in 330 msec 2024-11-12T18:15:24,734 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:15:24,734 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435324734"}]},"ts":"1731435324734"} 2024-11-12T18:15:24,735 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T18:15:24,736 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:15:24,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 810 msec 2024-11-12T18:15:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T18:15:25,070 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:25,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:25,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:25,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:25,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d07e2bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:25,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de70cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:25,075 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:25,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:25,078 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:15:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:15:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@15588bcd 2024-11-12T18:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:15:25,079 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:15:25,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:15:25,081 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:15:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742184_1360 (size=1097) 2024-11-12T18:15:25,499 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T18:15:25,499 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T18:15:25,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:15:25,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, REOPEN/MOVE}] 2024-11-12T18:15:25,503 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, REOPEN/MOVE 2024-11-12T18:15:25,504 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:25,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, REOPEN/MOVE because future has completed 2024-11-12T18:15:25,506 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:15:25,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:15:25,658 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,659 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:15:25,659 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 3946528656bca5567e369f3af446cf65, disabling compactions & flushes 2024-11-12T18:15:25,659 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,659 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,659 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. after waiting 0 ms 2024-11-12T18:15:25,659 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,662 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:15:25,663 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,663 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 3946528656bca5567e369f3af446cf65: Waiting for close lock at 1731435325659Running coprocessor pre-close hooks at 1731435325659Disabling compacts and flushes for region at 1731435325659Disabling writes for close at 1731435325659Writing region close event to WAL at 1731435325659Running coprocessor post-close hooks at 1731435325663 (+4 ms)Closed at 1731435325663 2024-11-12T18:15:25,663 WARN [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 3946528656bca5567e369f3af446cf65 to self. 2024-11-12T18:15:25,665 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,665 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=CLOSED 2024-11-12T18:15:25,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:15:25,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-12T18:15:25,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 in 162 msec 2024-11-12T18:15:25,670 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, REOPEN/MOVE; state=CLOSED, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=true 2024-11-12T18:15:25,821 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:25,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, REOPEN/MOVE because future has completed 2024-11-12T18:15:25,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:15:25,978 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,978 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:15:25,979 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,979 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:15:25,979 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,979 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,980 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,981 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:25,981 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3946528656bca5567e369f3af446cf65 columnFamilyName A 2024-11-12T18:15:25,982 DEBUG [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:25,983 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(327): Store=3946528656bca5567e369f3af446cf65/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:25,983 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,984 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:25,984 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3946528656bca5567e369f3af446cf65 columnFamilyName B 2024-11-12T18:15:25,984 DEBUG [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:25,985 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(327): Store=3946528656bca5567e369f3af446cf65/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:25,985 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,985 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:25,985 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3946528656bca5567e369f3af446cf65 columnFamilyName C 2024-11-12T18:15:25,986 DEBUG [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:25,986 INFO [StoreOpener-3946528656bca5567e369f3af446cf65-1 {}] regionserver.HStore(327): Store=3946528656bca5567e369f3af446cf65/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:25,986 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,986 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,987 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,988 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,988 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,989 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,989 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:15:25,990 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,991 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 3946528656bca5567e369f3af446cf65; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61115432, jitterRate=-0.08930909633636475}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:15:25,991 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:25,991 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 3946528656bca5567e369f3af446cf65: Running coprocessor pre-open hook at 1731435325979Writing region info on filesystem at 1731435325979Initializing all the Stores at 1731435325980 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435325980Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435325980Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435325980Cleaning up temporary data from old regions at 1731435325989 (+9 ms)Running coprocessor post-open hooks at 1731435325991 (+2 ms)Region opened successfully at 1731435325991 2024-11-12T18:15:25,992 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., pid=96, masterSystemTime=1731435325975 2024-11-12T18:15:25,994 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,994 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:25,994 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=OPEN, openSeqNum=5, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:25,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:15:25,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-12T18:15:25,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 in 173 msec 2024-11-12T18:15:25,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-12T18:15:25,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, REOPEN/MOVE in 495 msec 2024-11-12T18:15:26,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-12T18:15:26,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-12T18:15:26,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-12T18:15:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-12T18:15:26,004 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:26,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,005 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27819e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,007 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eaf01d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,009 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40344f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,010 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b1a666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7194042d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fafd5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,014 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62752f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86b6461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc2ec2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4f93b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583bfa39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a415af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,025 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378f2670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f179de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b90e21a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21937714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@316eeadb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,033 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cba1b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:26,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:26,034 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d39c280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:26,036 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432b9480, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:26,037 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,037 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,041 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,041 DEBUG [Thread-1644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,041 DEBUG [Thread-1648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,041 DEBUG [Thread-1639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,045 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,045 DEBUG [Thread-1633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-12T18:15:26,046 DEBUG [Thread-1637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,046 DEBUG [Thread-1646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,047 DEBUG [Thread-1642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,048 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:26,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:26,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:26,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:15:26,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d09369a185c24523b13188a92439cf52_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435326045/Put/seqid=0 2024-11-12T18:15:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742185_1361 (size=14594) 2024-11-12T18:15:26,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:26,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435326066, max timestamp=1731435326099Num uniques 859; 2024-11-12T18:15:26,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435326066, max timestamp=1731435326099Num uniques 859; 2024-11-12T18:15:26,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:26,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:26,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435326066, max timestamp=1731435326099Num uniques 859; 2024-11-12T18:15:26,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:26,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d09369a185c24523b13188a92439cf52_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d09369a185c24523b13188a92439cf52_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:26,110 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3c40ae7b22f240e9916ec36a88caa165, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:26,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3c40ae7b22f240e9916ec36a88caa165 is 175, key is test_row_0/A:col10/1731435326045/Put/seqid=0 2024-11-12T18:15:26,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742186_1362 (size=39549) 2024-11-12T18:15:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:26,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:26,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435386144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435386144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435386192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435386192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435386192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435386310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435386310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435386320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435386320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435386321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:26,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:26,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:26,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3c40ae7b22f240e9916ec36a88caa165 2024-11-12T18:15:26,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435386530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435386530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/0ae4c6a41c8d4cfd955e518e36914d8d is 50, key is test_row_0/B:col10/1731435326045/Put/seqid=0 2024-11-12T18:15:26,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742187_1363 (size=12001) 2024-11-12T18:15:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435386541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435386541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435386850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435386850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435386860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435386860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435386860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:26,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/0ae4c6a41c8d4cfd955e518e36914d8d 2024-11-12T18:15:26,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/e25bc85a82914726b4764d40fe4d7549 is 50, key is test_row_0/C:col10/1731435326045/Put/seqid=0 2024-11-12T18:15:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742188_1364 (size=12001) 2024-11-12T18:15:26,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:26,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:26,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:26,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:27,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:27,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:27,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:27,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:27,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:27,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435387370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435387370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:27,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/e25bc85a82914726b4764d40fe4d7549 2024-11-12T18:15:27,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3c40ae7b22f240e9916ec36a88caa165 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165 2024-11-12T18:15:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435387381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:27,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435387381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:27,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435387381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:27,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165, entries=200, sequenceid=14, filesize=38.6 K 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:27,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/0ae4c6a41c8d4cfd955e518e36914d8d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0ae4c6a41c8d4cfd955e518e36914d8d 2024-11-12T18:15:27,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0ae4c6a41c8d4cfd955e518e36914d8d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T18:15:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/e25bc85a82914726b4764d40fe4d7549 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e25bc85a82914726b4764d40fe4d7549 2024-11-12T18:15:27,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e25bc85a82914726b4764d40fe4d7549, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T18:15:27,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3946528656bca5567e369f3af446cf65 in 1351ms, sequenceid=14, compaction requested=false 2024-11-12T18:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:27,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T18:15:27,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:27,452 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T18:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:27,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111219db00ca47ed470f8ffd869ae3139f45_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435326152/Put/seqid=0 2024-11-12T18:15:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742189_1365 (size=12154) 2024-11-12T18:15:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:27,910 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111219db00ca47ed470f8ffd869ae3139f45_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111219db00ca47ed470f8ffd869ae3139f45_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:27,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c31624e3dfb47afa028c40584b47739, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c31624e3dfb47afa028c40584b47739 is 175, key is test_row_0/A:col10/1731435326152/Put/seqid=0 2024-11-12T18:15:27,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742190_1366 (size=30955) 2024-11-12T18:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:28,316 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c31624e3dfb47afa028c40584b47739 2024-11-12T18:15:28,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/7187aff9c49643fa8a395995ec9dca96 is 50, key is test_row_0/B:col10/1731435326152/Put/seqid=0 2024-11-12T18:15:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742191_1367 (size=12001) 2024-11-12T18:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:28,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435388411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435388411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,423 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in 3946528656bca5567e369f3af446cf65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435388412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435388414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435388422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435388530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435388530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435388530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435388550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/7187aff9c49643fa8a395995ec9dca96 2024-11-12T18:15:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6070f99865a7459fbf92ad9f9d15a61c is 50, key is test_row_0/C:col10/1731435326152/Put/seqid=0 2024-11-12T18:15:28,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742192_1368 (size=12001) 2024-11-12T18:15:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435388750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435388751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435388751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435388770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,012 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:15:29,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435389070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435389071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435389073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435389090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,160 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6070f99865a7459fbf92ad9f9d15a61c 2024-11-12T18:15:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c31624e3dfb47afa028c40584b47739 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739 2024-11-12T18:15:29,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739, entries=150, sequenceid=50, filesize=30.2 K 2024-11-12T18:15:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/7187aff9c49643fa8a395995ec9dca96 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/7187aff9c49643fa8a395995ec9dca96 2024-11-12T18:15:29,176 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/7187aff9c49643fa8a395995ec9dca96, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T18:15:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6070f99865a7459fbf92ad9f9d15a61c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6070f99865a7459fbf92ad9f9d15a61c 2024-11-12T18:15:29,182 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6070f99865a7459fbf92ad9f9d15a61c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T18:15:29,185 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3946528656bca5567e369f3af446cf65 in 1733ms, sequenceid=50, compaction requested=false 2024-11-12T18:15:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T18:15:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-12T18:15:29,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T18:15:29,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1370 sec 2024-11-12T18:15:29,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.1450 sec 2024-11-12T18:15:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:29,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:29,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:29,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:29,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:29,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:29,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129824b4a4f396436296a81dbfd006a096_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435328420/Put/seqid=0 2024-11-12T18:15:29,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742193_1369 (size=12154) 2024-11-12T18:15:29,613 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:29,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129824b4a4f396436296a81dbfd006a096_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129824b4a4f396436296a81dbfd006a096_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:29,621 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6ef74fef4ad4436ba11522eb34489ad, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:29,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6ef74fef4ad4436ba11522eb34489ad is 175, key is test_row_0/A:col10/1731435328420/Put/seqid=0 2024-11-12T18:15:29,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742194_1370 (size=30955) 2024-11-12T18:15:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435389648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435329630, max timestamp=1731435329661Num uniques 859; 2024-11-12T18:15:29,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:29,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435329630, max timestamp=1731435329661Num uniques 859; 2024-11-12T18:15:29,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435329630, max timestamp=1731435329661Num uniques 859; 2024-11-12T18:15:29,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:29,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:29,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435389719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435389721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435389727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435389770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435389840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435389850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435389850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435390000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6ef74fef4ad4436ba11522eb34489ad 2024-11-12T18:15:30,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/12bd03202bc6411297c25110375d42cb is 50, key is test_row_0/B:col10/1731435328420/Put/seqid=0 2024-11-12T18:15:30,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742195_1371 (size=12001) 2024-11-12T18:15:30,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435390060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435390070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435390071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T18:15:30,200 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:30,201 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T18:15:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:30,205 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:30,206 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:30,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:30,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435390311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:30,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:30,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435390381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435390390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435390393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/12bd03202bc6411297c25110375d42cb 2024-11-12T18:15:30,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/954c39a213a446a48f6cfa298ff6819e is 50, key is test_row_0/C:col10/1731435328420/Put/seqid=0 2024-11-12T18:15:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435390440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,456 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in 3946528656bca5567e369f3af446cf65 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742196_1372 (size=12001) 2024-11-12T18:15:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:30,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:30,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:30,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:30,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:30,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:30,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435390830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/954c39a213a446a48f6cfa298ff6819e 2024-11-12T18:15:30,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6ef74fef4ad4436ba11522eb34489ad as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad 2024-11-12T18:15:30,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad, entries=150, sequenceid=61, filesize=30.2 K 2024-11-12T18:15:30,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/12bd03202bc6411297c25110375d42cb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/12bd03202bc6411297c25110375d42cb 2024-11-12T18:15:30,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/12bd03202bc6411297c25110375d42cb, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T18:15:30,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/954c39a213a446a48f6cfa298ff6819e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/954c39a213a446a48f6cfa298ff6819e 2024-11-12T18:15:30,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/954c39a213a446a48f6cfa298ff6819e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T18:15:30,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3946528656bca5567e369f3af446cf65 in 1294ms, sequenceid=61, compaction requested=true 2024-11-12T18:15:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:30,891 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:30,891 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:30,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:30,893 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,893 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,893 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=99.1 K 2024-11-12T18:15:30,893 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0ae4c6a41c8d4cfd955e518e36914d8d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/7187aff9c49643fa8a395995ec9dca96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/12bd03202bc6411297c25110375d42cb] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=35.2 K 2024-11-12T18:15:30,893 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad] 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae4c6a41c8d4cfd955e518e36914d8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731435326045 2024-11-12T18:15:30,893 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c40ae7b22f240e9916ec36a88caa165, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731435326045 2024-11-12T18:15:30,894 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7187aff9c49643fa8a395995ec9dca96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731435326142 2024-11-12T18:15:30,894 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c31624e3dfb47afa028c40584b47739, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731435326142 2024-11-12T18:15:30,894 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 12bd03202bc6411297c25110375d42cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435328412 2024-11-12T18:15:30,894 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ef74fef4ad4436ba11522eb34489ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435328412 2024-11-12T18:15:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:30,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-12T18:15:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:30,920 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:30,934 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:30,934 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/bd9678cc6bcb4463a969740633c18c29 is 50, key is test_row_0/B:col10/1731435328420/Put/seqid=0 2024-11-12T18:15:30,937 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111254902b3f615945ada0b9be3003b19575_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:30,939 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111254902b3f615945ada0b9be3003b19575_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:30,940 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111254902b3f615945ada0b9be3003b19575_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:30,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111243f600d3d6ea45e58ecaad52fd3ead14_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435329723/Put/seqid=0 2024-11-12T18:15:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435390947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435390949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435390951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742197_1373 (size=12104) 2024-11-12T18:15:30,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:30,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:30,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:30,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:30,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742198_1374 (size=4469) 2024-11-12T18:15:31,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742199_1375 (size=14594) 2024-11-12T18:15:31,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:31,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111243f600d3d6ea45e58ecaad52fd3ead14_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111243f600d3d6ea45e58ecaad52fd3ead14_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:31,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/88cab1145e0c46e4892e3061345c10e7, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:31,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/88cab1145e0c46e4892e3061345c10e7 is 175, key is test_row_0/A:col10/1731435329723/Put/seqid=0 2024-11-12T18:15:31,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742200_1376 (size=39549) 2024-11-12T18:15:31,042 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/88cab1145e0c46e4892e3061345c10e7 2024-11-12T18:15:31,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/61a7a03d5e8b4e0ab07fa36bbd1df64d is 50, key is test_row_0/B:col10/1731435329723/Put/seqid=0 2024-11-12T18:15:31,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742201_1377 (size=12001) 2024-11-12T18:15:31,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/61a7a03d5e8b4e0ab07fa36bbd1df64d 2024-11-12T18:15:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435391071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435391071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435391071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/732366763ebe4ac184d5102ba259431c is 50, key is test_row_0/C:col10/1731435329723/Put/seqid=0 2024-11-12T18:15:31,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742202_1378 (size=12001) 2024-11-12T18:15:31,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/732366763ebe4ac184d5102ba259431c 2024-11-12T18:15:31,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/88cab1145e0c46e4892e3061345c10e7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7 2024-11-12T18:15:31,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7, entries=200, sequenceid=96, filesize=38.6 K 2024-11-12T18:15:31,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/61a7a03d5e8b4e0ab07fa36bbd1df64d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/61a7a03d5e8b4e0ab07fa36bbd1df64d 2024-11-12T18:15:31,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/61a7a03d5e8b4e0ab07fa36bbd1df64d, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T18:15:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/732366763ebe4ac184d5102ba259431c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/732366763ebe4ac184d5102ba259431c 2024-11-12T18:15:31,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/732366763ebe4ac184d5102ba259431c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-12T18:15:31,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 3946528656bca5567e369f3af446cf65 in 250ms, sequenceid=96, compaction requested=true 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:31,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-12T18:15:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:31,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:31,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:31,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:31,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111253b381198d9548f4ae87dc49d66b1a75_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435330944/Put/seqid=0 2024-11-12T18:15:31,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742203_1379 (size=14594) 2024-11-12T18:15:31,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:31,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111253b381198d9548f4ae87dc49d66b1a75_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111253b381198d9548f4ae87dc49d66b1a75_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:31,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2bf68a5e99644d4a8c9ad17b1b5140da, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:31,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2bf68a5e99644d4a8c9ad17b1b5140da is 175, key is test_row_0/A:col10/1731435330944/Put/seqid=0 2024-11-12T18:15:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742204_1380 (size=39549) 2024-11-12T18:15:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:31,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2bf68a5e99644d4a8c9ad17b1b5140da 2024-11-12T18:15:31,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/15b30e1a1c9547a6896ebbf505f97f8a is 50, key is test_row_0/B:col10/1731435330944/Put/seqid=0 2024-11-12T18:15:31,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742205_1381 (size=12001) 2024-11-12T18:15:31,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/15b30e1a1c9547a6896ebbf505f97f8a 2024-11-12T18:15:31,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/9266fd5101464273834add093eee5806 is 50, key is test_row_0/C:col10/1731435330944/Put/seqid=0 2024-11-12T18:15:31,384 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/bd9678cc6bcb4463a969740633c18c29 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/bd9678cc6bcb4463a969740633c18c29 2024-11-12T18:15:31,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742206_1382 (size=12001) 2024-11-12T18:15:31,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/9266fd5101464273834add093eee5806 2024-11-12T18:15:31,391 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into bd9678cc6bcb4463a969740633c18c29(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:31,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:31,391 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=13, startTime=1731435330891; duration=0sec 2024-11-12T18:15:31,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-12T18:15:31,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:31,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-12T18:15:31,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:15:31,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:15:31,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. because compaction request was cancelled 2024-11-12T18:15:31,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:31,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:31,394 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:31,394 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:31,394 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,394 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e25bc85a82914726b4764d40fe4d7549, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6070f99865a7459fbf92ad9f9d15a61c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/954c39a213a446a48f6cfa298ff6819e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/732366763ebe4ac184d5102ba259431c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=46.9 K 2024-11-12T18:15:31,396 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e25bc85a82914726b4764d40fe4d7549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731435326045 2024-11-12T18:15:31,397 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6070f99865a7459fbf92ad9f9d15a61c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731435326142 2024-11-12T18:15:31,397 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 954c39a213a446a48f6cfa298ff6819e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435328412 2024-11-12T18:15:31,397 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#320 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:31,398 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/8297c6b16c554ee0a3d3f5d1c42c9a2d is 175, key is test_row_0/A:col10/1731435328420/Put/seqid=0 2024-11-12T18:15:31,398 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 732366763ebe4ac184d5102ba259431c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435329717 2024-11-12T18:15:31,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2bf68a5e99644d4a8c9ad17b1b5140da as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da 2024-11-12T18:15:31,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435331359, max timestamp=1731435331402Num uniques 859; 2024-11-12T18:15:31,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:31,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435331359, max timestamp=1731435331402Num uniques 859; 2024-11-12T18:15:31,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:31,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435331359, max timestamp=1731435331402Num uniques 859; 2024-11-12T18:15:31,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:31,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da, entries=200, sequenceid=107, filesize=38.6 K 2024-11-12T18:15:31,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/15b30e1a1c9547a6896ebbf505f97f8a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/15b30e1a1c9547a6896ebbf505f97f8a 2024-11-12T18:15:31,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742207_1383 (size=31058) 2024-11-12T18:15:31,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/15b30e1a1c9547a6896ebbf505f97f8a, entries=150, sequenceid=107, filesize=11.7 K 2024-11-12T18:15:31,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/9266fd5101464273834add093eee5806 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/9266fd5101464273834add093eee5806 2024-11-12T18:15:31,427 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:31,428 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6b10e3eb16314d638018e5d0bd7c93dd is 50, key is test_row_0/C:col10/1731435329723/Put/seqid=0 2024-11-12T18:15:31,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/9266fd5101464273834add093eee5806, entries=150, sequenceid=107, filesize=11.7 K 2024-11-12T18:15:31,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=60.38 KB/61830 for 3946528656bca5567e369f3af446cf65 in 137ms, sequenceid=107, compaction requested=true 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:31,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-12T18:15:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:31,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=73.80 KB heapSize=196.31 KB 2024-11-12T18:15:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:31,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742208_1384 (size=12139) 2024-11-12T18:15:31,450 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6b10e3eb16314d638018e5d0bd7c93dd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6b10e3eb16314d638018e5d0bd7c93dd 2024-11-12T18:15:31,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,459 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into 6b10e3eb16314d638018e5d0bd7c93dd(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:31,459 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:31,459 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=12, startTime=1731435331157; duration=0sec 2024-11-12T18:15:31,459 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-12T18:15:31,459 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:31,459 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-12T18:15:31,460 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:15:31,460 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. because compaction request was cancelled 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:31,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:31,462 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,462 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/bd9678cc6bcb4463a969740633c18c29, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/61a7a03d5e8b4e0ab07fa36bbd1df64d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/15b30e1a1c9547a6896ebbf505f97f8a] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=35.3 K 2024-11-12T18:15:31,462 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9678cc6bcb4463a969740633c18c29, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435328412 2024-11-12T18:15:31,462 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 61a7a03d5e8b4e0ab07fa36bbd1df64d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435329717 2024-11-12T18:15:31,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111256bfeafb1302411b8d758e886d91c935_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435331436/Put/seqid=0 2024-11-12T18:15:31,466 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b30e1a1c9547a6896ebbf505f97f8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731435330924 2024-11-12T18:15:31,501 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:31,502 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/47e12ae99e774053862ccdb6192891b4 is 50, key is test_row_0/B:col10/1731435330944/Put/seqid=0 2024-11-12T18:15:31,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742209_1385 (size=14794) 2024-11-12T18:15:31,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742210_1386 (size=12207) 2024-11-12T18:15:31,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435391564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435391576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435391576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:31,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435391690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435391690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435391690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,828 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/8297c6b16c554ee0a3d3f5d1c42c9a2d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/8297c6b16c554ee0a3d3f5d1c42c9a2d 2024-11-12T18:15:31,834 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into 8297c6b16c554ee0a3d3f5d1c42c9a2d(size=30.3 K), total size for store is 107.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:31,834 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=13, startTime=1731435330890; duration=0sec 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 3 compacting, 0 eligible, 16 blocking 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. because compaction request was cancelled 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:31,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:15:31,835 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:15:31,835 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:15:31,835 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. because compaction request was cancelled 2024-11-12T18:15:31,835 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:31,835 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:15:31,837 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:15:31,837 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:15:31,837 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. because compaction request was cancelled 2024-11-12T18:15:31,837 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435391861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,878 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in 3946528656bca5567e369f3af446cf65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435391900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435391900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:31,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111256bfeafb1302411b8d758e886d91c935_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111256bfeafb1302411b8d758e886d91c935_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:31,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c9a9be3882574e28b8c6172c5f231c71, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:31,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c9a9be3882574e28b8c6172c5f231c71 is 175, key is test_row_0/A:col10/1731435331436/Put/seqid=0 2024-11-12T18:15:31,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742211_1387 (size=39749) 2024-11-12T18:15:31,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:31,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c9a9be3882574e28b8c6172c5f231c71 2024-11-12T18:15:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:31,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435391910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:31,932 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/47e12ae99e774053862ccdb6192891b4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/47e12ae99e774053862ccdb6192891b4 2024-11-12T18:15:31,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/6bf96a42055e420988f265aec31d08c8 is 50, key is test_row_0/B:col10/1731435331436/Put/seqid=0 2024-11-12T18:15:31,937 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into 47e12ae99e774053862ccdb6192891b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:31,937 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:31,937 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=13, startTime=1731435331157; duration=0sec 2024-11-12T18:15:31,937 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:31,937 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:31,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742212_1388 (size=12151) 2024-11-12T18:15:31,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/6bf96a42055e420988f265aec31d08c8 2024-11-12T18:15:31,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/2742bcf231e048758557d3d0a7e5e16d is 50, key is test_row_0/C:col10/1731435331436/Put/seqid=0 2024-11-12T18:15:31,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742213_1389 (size=12151) 2024-11-12T18:15:32,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435392210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435392210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435392230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:32,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/2742bcf231e048758557d3d0a7e5e16d 2024-11-12T18:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c9a9be3882574e28b8c6172c5f231c71 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71 2024-11-12T18:15:32,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71, entries=200, sequenceid=135, filesize=38.8 K 2024-11-12T18:15:32,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:32,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/6bf96a42055e420988f265aec31d08c8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/6bf96a42055e420988f265aec31d08c8 2024-11-12T18:15:32,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/6bf96a42055e420988f265aec31d08c8, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T18:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/2742bcf231e048758557d3d0a7e5e16d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/2742bcf231e048758557d3d0a7e5e16d 2024-11-12T18:15:32,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/2742bcf231e048758557d3d0a7e5e16d, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T18:15:32,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~216.19 KB/221376, currentSize=107.34 KB/109920 for 3946528656bca5567e369f3af446cf65 in 979ms, sequenceid=135, compaction requested=true 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T18:15:32,418 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:32,418 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:32,420 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:32,420 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:32,420 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,420 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6b10e3eb16314d638018e5d0bd7c93dd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/9266fd5101464273834add093eee5806, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/2742bcf231e048758557d3d0a7e5e16d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=35.4 K 2024-11-12T18:15:32,420 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149905 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:32,420 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:32,420 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,421 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/8297c6b16c554ee0a3d3f5d1c42c9a2d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=146.4 K 2024-11-12T18:15:32,421 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,421 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/8297c6b16c554ee0a3d3f5d1c42c9a2d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71] 2024-11-12T18:15:32,421 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b10e3eb16314d638018e5d0bd7c93dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435329717 2024-11-12T18:15:32,421 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8297c6b16c554ee0a3d3f5d1c42c9a2d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731435328412 2024-11-12T18:15:32,422 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9266fd5101464273834add093eee5806, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731435330924 2024-11-12T18:15:32,422 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 2742bcf231e048758557d3d0a7e5e16d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731435331420 2024-11-12T18:15:32,422 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88cab1145e0c46e4892e3061345c10e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731435329705 2024-11-12T18:15:32,423 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf68a5e99644d4a8c9ad17b1b5140da, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731435330924 2024-11-12T18:15:32,423 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a9be3882574e28b8c6172c5f231c71, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731435331412 2024-11-12T18:15:32,436 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:32,437 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/be9aa15cd29f447d86de2a83160abe84 is 50, key is test_row_0/C:col10/1731435331436/Put/seqid=0 2024-11-12T18:15:32,447 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:32,455 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112a00d3963a52244a9b6eea5a4f9dcbb0e_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:32,459 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112a00d3963a52244a9b6eea5a4f9dcbb0e_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:32,459 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a00d3963a52244a9b6eea5a4f9dcbb0e_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:32,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742214_1390 (size=12391) 2024-11-12T18:15:32,486 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/be9aa15cd29f447d86de2a83160abe84 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/be9aa15cd29f447d86de2a83160abe84 2024-11-12T18:15:32,495 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into be9aa15cd29f447d86de2a83160abe84(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:32,495 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:32,495 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=13, startTime=1731435332418; duration=0sec 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. because compaction request was cancelled 2024-11-12T18:15:32,496 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:32,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742215_1391 (size=4469) 2024-11-12T18:15:32,500 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#334 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:32,500 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/ea53a5942bab4be6b06fb25857341a33 is 175, key is test_row_0/A:col10/1731435331436/Put/seqid=0 2024-11-12T18:15:32,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T18:15:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:32,538 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:15:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:32,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742216_1392 (size=31345) 2024-11-12T18:15:32,557 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/ea53a5942bab4be6b06fb25857341a33 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ea53a5942bab4be6b06fb25857341a33 2024-11-12T18:15:32,566 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into ea53a5942bab4be6b06fb25857341a33(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:32,566 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:32,566 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=12, startTime=1731435332418; duration=0sec 2024-11-12T18:15:32,567 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:32,567 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:32,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112365c8bcc41494b069d9f78ffc7369432_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435331574/Put/seqid=0 2024-11-12T18:15:32,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742217_1393 (size=12304) 2024-11-12T18:15:32,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435392780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435392780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435392782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435392900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435392901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435392901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:33,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112365c8bcc41494b069d9f78ffc7369432_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112365c8bcc41494b069d9f78ffc7369432_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:33,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2a82627d38854d8e8313bb1372e4d852, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:33,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2a82627d38854d8e8313bb1372e4d852 is 175, key is test_row_0/A:col10/1731435331574/Put/seqid=0 2024-11-12T18:15:33,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742218_1394 (size=31105) 2024-11-12T18:15:33,077 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2a82627d38854d8e8313bb1372e4d852 2024-11-12T18:15:33,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/85893ea7c6634bd18b1a85311011c194 is 50, key is test_row_0/B:col10/1731435331574/Put/seqid=0 2024-11-12T18:15:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742219_1395 (size=12151) 2024-11-12T18:15:33,132 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/85893ea7c6634bd18b1a85311011c194 2024-11-12T18:15:33,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435393131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435393131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435393132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/0192a76138c7440a9e5605db2dd7b1b9 is 50, key is test_row_0/C:col10/1731435331574/Put/seqid=0 2024-11-12T18:15:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742220_1396 (size=12151) 2024-11-12T18:15:33,215 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/0192a76138c7440a9e5605db2dd7b1b9 2024-11-12T18:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/2a82627d38854d8e8313bb1372e4d852 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852 2024-11-12T18:15:33,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852, entries=150, sequenceid=158, filesize=30.4 K 2024-11-12T18:15:33,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/85893ea7c6634bd18b1a85311011c194 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/85893ea7c6634bd18b1a85311011c194 2024-11-12T18:15:33,233 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/85893ea7c6634bd18b1a85311011c194, entries=150, sequenceid=158, filesize=11.9 K 2024-11-12T18:15:33,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/0192a76138c7440a9e5605db2dd7b1b9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/0192a76138c7440a9e5605db2dd7b1b9 2024-11-12T18:15:33,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/0192a76138c7440a9e5605db2dd7b1b9, entries=150, sequenceid=158, filesize=11.9 K 2024-11-12T18:15:33,242 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3946528656bca5567e369f3af446cf65 in 704ms, sequenceid=158, compaction requested=true 2024-11-12T18:15:33,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:33,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:33,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T18:15:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T18:15:33,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T18:15:33,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0400 sec 2024-11-12T18:15:33,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.0500 sec 2024-11-12T18:15:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:33,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:15:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:33,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112eb466ed1f6f8465d88c15853cc32d01f_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:33,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742221_1397 (size=12304) 2024-11-12T18:15:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435393515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435393516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435393520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435393633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435393641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435393641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435393850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435393850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435393851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:33,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:33,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112eb466ed1f6f8465d88c15853cc32d01f_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eb466ed1f6f8465d88c15853cc32d01f_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:33,891 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/805cc704cf624e01a337e4dae407a399, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:33,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/805cc704cf624e01a337e4dae407a399 is 175, key is test_row_0/A:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:33,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742222_1398 (size=31105) 2024-11-12T18:15:33,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435393890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:33,901 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in 3946528656bca5567e369f3af446cf65 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435394170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435394171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435394171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/805cc704cf624e01a337e4dae407a399 2024-11-12T18:15:34,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/897f9c18cc554d338aa13974d8159e59 is 50, key is test_row_0/B:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T18:15:34,361 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:34,362 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:34,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742223_1399 (size=12151) 2024-11-12T18:15:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-12T18:15:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:34,369 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:34,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/897f9c18cc554d338aa13974d8159e59 2024-11-12T18:15:34,370 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:34,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:34,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/a6617d611c5f4f9ab8077d68d3226b0f is 50, key is test_row_0/C:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:34,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742224_1400 (size=12151) 2024-11-12T18:15:34,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/a6617d611c5f4f9ab8077d68d3226b0f 2024-11-12T18:15:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/805cc704cf624e01a337e4dae407a399 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399 2024-11-12T18:15:34,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399, entries=150, sequenceid=173, filesize=30.4 K 2024-11-12T18:15:34,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/897f9c18cc554d338aa13974d8159e59 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/897f9c18cc554d338aa13974d8159e59 2024-11-12T18:15:34,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/897f9c18cc554d338aa13974d8159e59, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T18:15:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/a6617d611c5f4f9ab8077d68d3226b0f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a6617d611c5f4f9ab8077d68d3226b0f 2024-11-12T18:15:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a6617d611c5f4f9ab8077d68d3226b0f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T18:15:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3946528656bca5567e369f3af446cf65 in 987ms, sequenceid=173, compaction requested=true 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:34,452 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:15:34,452 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:34,453 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:34,453 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:34,453 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,453 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ea53a5942bab4be6b06fb25857341a33, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=91.4 K 2024-11-12T18:15:34,453 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,453 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ea53a5942bab4be6b06fb25857341a33, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399] 2024-11-12T18:15:34,457 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea53a5942bab4be6b06fb25857341a33, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731435331420 2024-11-12T18:15:34,458 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:34,458 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:34,458 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,458 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/47e12ae99e774053862ccdb6192891b4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/6bf96a42055e420988f265aec31d08c8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/85893ea7c6634bd18b1a85311011c194, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/897f9c18cc554d338aa13974d8159e59] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=47.5 K 2024-11-12T18:15:34,458 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a82627d38854d8e8313bb1372e4d852, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435331547 2024-11-12T18:15:34,458 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 47e12ae99e774053862ccdb6192891b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731435330924 2024-11-12T18:15:34,459 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 805cc704cf624e01a337e4dae407a399, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731435332764 2024-11-12T18:15:34,459 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf96a42055e420988f265aec31d08c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731435331420 2024-11-12T18:15:34,459 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 85893ea7c6634bd18b1a85311011c194, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435331547 2024-11-12T18:15:34,460 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 897f9c18cc554d338aa13974d8159e59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731435332764 2024-11-12T18:15:34,478 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:34,487 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:34,488 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/2beb238e30d149a3a80e6686f53b8021 is 50, key is test_row_0/B:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:34,494 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ba7c2165484e47e19951cfe95d04e0be_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:34,496 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ba7c2165484e47e19951cfe95d04e0be_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:34,496 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ba7c2165484e47e19951cfe95d04e0be_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:34,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:15:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:34,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:34,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742225_1401 (size=12493) 2024-11-12T18:15:34,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742226_1402 (size=4469) 2024-11-12T18:15:34,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120551443e63404344b834316d48d69934_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435334509/Put/seqid=0 2024-11-12T18:15:34,621 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#341 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:34,622 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/9b06a5736414461fb411c5a2b1756a0a is 175, key is test_row_0/A:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:34,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:34,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:34,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:34,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742227_1403 (size=14794) 2024-11-12T18:15:34,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742228_1404 (size=31447) 2024-11-12T18:15:34,732 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/9b06a5736414461fb411c5a2b1756a0a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/9b06a5736414461fb411c5a2b1756a0a 2024-11-12T18:15:34,738 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into 9b06a5736414461fb411c5a2b1756a0a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:34,738 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:34,738 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=13, startTime=1731435334452; duration=0sec 2024-11-12T18:15:34,739 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:34,739 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:34,739 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435394715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,743 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:34,744 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:34,744 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,744 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/be9aa15cd29f447d86de2a83160abe84, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/0192a76138c7440a9e5605db2dd7b1b9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a6617d611c5f4f9ab8077d68d3226b0f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=35.8 K 2024-11-12T18:15:34,746 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9aa15cd29f447d86de2a83160abe84, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731435331420 2024-11-12T18:15:34,748 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0192a76138c7440a9e5605db2dd7b1b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435331547 2024-11-12T18:15:34,748 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6617d611c5f4f9ab8077d68d3226b0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731435332764 2024-11-12T18:15:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435394728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435394730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435394732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,771 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:34,771 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/af06de348b854350bf36870014143360 is 50, key is test_row_0/C:col10/1731435332764/Put/seqid=0 2024-11-12T18:15:34,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742229_1405 (size=12493) 2024-11-12T18:15:34,827 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/af06de348b854350bf36870014143360 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/af06de348b854350bf36870014143360 2024-11-12T18:15:34,832 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into af06de348b854350bf36870014143360(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:34,832 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:34,832 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=13, startTime=1731435334452; duration=0sec 2024-11-12T18:15:34,833 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:34,833 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:34,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:34,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:34,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435394851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435394871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435394860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435394871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:34,972 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/2beb238e30d149a3a80e6686f53b8021 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/2beb238e30d149a3a80e6686f53b8021 2024-11-12T18:15:34,981 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into 2beb238e30d149a3a80e6686f53b8021(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:34,981 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:34,981 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=12, startTime=1731435334452; duration=0sec 2024-11-12T18:15:34,981 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:34,981 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:34,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:35,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:35,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120551443e63404344b834316d48d69934_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120551443e63404344b834316d48d69934_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:35,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6f62dc41cc64745a05ff45f955641d3, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:35,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6f62dc41cc64745a05ff45f955641d3 is 175, key is test_row_0/A:col10/1731435334509/Put/seqid=0 2024-11-12T18:15:35,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435395100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435395101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435395101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435395101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742230_1406 (size=39749) 2024-11-12T18:15:35,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435395430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435395430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435395430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435395430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:35,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6f62dc41cc64745a05ff45f955641d3 2024-11-12T18:15:35,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8be811ea0bc246fb8aa3c5cb8e09a3fe is 50, key is test_row_0/B:col10/1731435334509/Put/seqid=0 2024-11-12T18:15:35,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742231_1407 (size=12151) 2024-11-12T18:15:35,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:35,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435395954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435395955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435395955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:35,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435395962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:36,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8be811ea0bc246fb8aa3c5cb8e09a3fe 2024-11-12T18:15:36,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/76df9ab48693416aa317e629bea88b57 is 50, key is test_row_0/C:col10/1731435334509/Put/seqid=0 2024-11-12T18:15:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742232_1408 (size=12151) 2024-11-12T18:15:36,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/76df9ab48693416aa317e629bea88b57 2024-11-12T18:15:36,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:36,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:36,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:36,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/e6f62dc41cc64745a05ff45f955641d3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3 2024-11-12T18:15:36,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3, entries=200, sequenceid=192, filesize=38.8 K 2024-11-12T18:15:36,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8be811ea0bc246fb8aa3c5cb8e09a3fe as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8be811ea0bc246fb8aa3c5cb8e09a3fe 2024-11-12T18:15:36,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8be811ea0bc246fb8aa3c5cb8e09a3fe, entries=150, sequenceid=192, filesize=11.9 K 2024-11-12T18:15:36,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/76df9ab48693416aa317e629bea88b57 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/76df9ab48693416aa317e629bea88b57 2024-11-12T18:15:36,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/76df9ab48693416aa317e629bea88b57, entries=150, sequenceid=192, filesize=11.9 K 2024-11-12T18:15:36,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3946528656bca5567e369f3af446cf65 in 1621ms, sequenceid=192, compaction requested=false 2024-11-12T18:15:36,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:36,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T18:15:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:36,239 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:15:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:36,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126d9b8a8fe49f42aa985ab3888530350c_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435334730/Put/seqid=0 2024-11-12T18:15:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742233_1409 (size=12304) 2024-11-12T18:15:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:36,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:36,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126d9b8a8fe49f42aa985ab3888530350c_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d9b8a8fe49f42aa985ab3888530350c_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:36,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/ae8995c09f744633aa49fa7708b60548, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:36,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/ae8995c09f744633aa49fa7708b60548 is 175, key is test_row_0/A:col10/1731435334730/Put/seqid=0 2024-11-12T18:15:36,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742234_1410 (size=31105) 2024-11-12T18:15:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:37,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435397044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435397046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435397049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435397058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,094 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/ae8995c09f744633aa49fa7708b60548 2024-11-12T18:15:37,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/aa70ed4792574f1eb33ad9c21866ab43 is 50, key is test_row_0/B:col10/1731435334730/Put/seqid=0 2024-11-12T18:15:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742235_1411 (size=12151) 2024-11-12T18:15:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435397161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435397161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435397171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435397181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435397381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435397388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435397391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435397401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,526 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/aa70ed4792574f1eb33ad9c21866ab43 2024-11-12T18:15:37,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/985987de29c24f8b9a84ed2e9e4aeeaf is 50, key is test_row_0/C:col10/1731435334730/Put/seqid=0 2024-11-12T18:15:37,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742236_1412 (size=12151) 2024-11-12T18:15:37,598 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/985987de29c24f8b9a84ed2e9e4aeeaf 2024-11-12T18:15:37,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/ae8995c09f744633aa49fa7708b60548 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548 2024-11-12T18:15:37,614 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548, entries=150, sequenceid=208, filesize=30.4 K 2024-11-12T18:15:37,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/aa70ed4792574f1eb33ad9c21866ab43 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/aa70ed4792574f1eb33ad9c21866ab43 2024-11-12T18:15:37,623 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/aa70ed4792574f1eb33ad9c21866ab43, entries=150, sequenceid=208, filesize=11.9 K 2024-11-12T18:15:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/985987de29c24f8b9a84ed2e9e4aeeaf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/985987de29c24f8b9a84ed2e9e4aeeaf 2024-11-12T18:15:37,650 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/985987de29c24f8b9a84ed2e9e4aeeaf, entries=150, sequenceid=208, filesize=11.9 K 2024-11-12T18:15:37,653 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3946528656bca5567e369f3af446cf65 in 1413ms, sequenceid=208, compaction requested=true 2024-11-12T18:15:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-12T18:15:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-12T18:15:37,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T18:15:37,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2850 sec 2024-11-12T18:15:37,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.2970 sec 2024-11-12T18:15:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:37,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:15:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:37,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:37,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127453196a039b4414bec133b1cd276f4e_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:37,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742237_1413 (size=17284) 2024-11-12T18:15:37,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:37,788 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127453196a039b4414bec133b1cd276f4e_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127453196a039b4414bec133b1cd276f4e_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:37,789 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3a15be8de2014b8eb9a1ddbcd145905e, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:37,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3a15be8de2014b8eb9a1ddbcd145905e is 175, key is test_row_0/A:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:37,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435397785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435397785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435397788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435397786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742238_1414 (size=48389) 2024-11-12T18:15:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35016 deadline: 1731435397910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435397910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,930 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in 3946528656bca5567e369f3af446cf65 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:15:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435397921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435397933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:37,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435397932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435398144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435398152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435398175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435398181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3a15be8de2014b8eb9a1ddbcd145905e 2024-11-12T18:15:38,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/c7708e8e75f44060a040a2f720ab7f85 is 50, key is test_row_0/B:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:38,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742239_1415 (size=12151) 2024-11-12T18:15:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435398461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435398484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435398501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435398502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T18:15:38,553 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:38,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-12T18:15:38,562 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T18:15:38,563 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:38,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:38,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/c7708e8e75f44060a040a2f720ab7f85 2024-11-12T18:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T18:15:38,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/1754b0aad0154ef4a366d37a2e79cd8e is 50, key is test_row_0/C:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:38,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T18:15:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:38,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:38,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742240_1416 (size=12151) 2024-11-12T18:15:38,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T18:15:38,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:38,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:38,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:38,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:38,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T18:15:38,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435398984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:39,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435399012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:39,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:39,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T18:15:39,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:39,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:39,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:39,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435399030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:39,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:39,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435399031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:39,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:39,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/1754b0aad0154ef4a366d37a2e79cd8e 2024-11-12T18:15:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T18:15:39,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T18:15:39,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:39,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:39,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:39,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/3a15be8de2014b8eb9a1ddbcd145905e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e 2024-11-12T18:15:39,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e, entries=250, sequenceid=229, filesize=47.3 K 2024-11-12T18:15:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/c7708e8e75f44060a040a2f720ab7f85 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/c7708e8e75f44060a040a2f720ab7f85 2024-11-12T18:15:39,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/c7708e8e75f44060a040a2f720ab7f85, entries=150, sequenceid=229, filesize=11.9 K 2024-11-12T18:15:39,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/1754b0aad0154ef4a366d37a2e79cd8e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/1754b0aad0154ef4a366d37a2e79cd8e 2024-11-12T18:15:39,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/1754b0aad0154ef4a366d37a2e79cd8e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-12T18:15:39,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3946528656bca5567e369f3af446cf65 in 1529ms, sequenceid=229, compaction requested=true 2024-11-12T18:15:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:39,253 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:39,254 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:39,267 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150690 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:39,267 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:39,267 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,268 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/9b06a5736414461fb411c5a2b1756a0a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=147.2 K 2024-11-12T18:15:39,268 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,268 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/9b06a5736414461fb411c5a2b1756a0a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e] 2024-11-12T18:15:39,268 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:39,269 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:39,269 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,269 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/2beb238e30d149a3a80e6686f53b8021, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8be811ea0bc246fb8aa3c5cb8e09a3fe, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/aa70ed4792574f1eb33ad9c21866ab43, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/c7708e8e75f44060a040a2f720ab7f85] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=47.8 K 2024-11-12T18:15:39,269 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b06a5736414461fb411c5a2b1756a0a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731435332764 2024-11-12T18:15:39,269 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 2beb238e30d149a3a80e6686f53b8021, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731435332764 2024-11-12T18:15:39,269 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f62dc41cc64745a05ff45f955641d3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731435333501 2024-11-12T18:15:39,270 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8be811ea0bc246fb8aa3c5cb8e09a3fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731435333501 2024-11-12T18:15:39,270 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae8995c09f744633aa49fa7708b60548, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731435334674 2024-11-12T18:15:39,270 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting aa70ed4792574f1eb33ad9c21866ab43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731435334674 2024-11-12T18:15:39,270 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a15be8de2014b8eb9a1ddbcd145905e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731435337028 2024-11-12T18:15:39,271 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c7708e8e75f44060a040a2f720ab7f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731435337047 2024-11-12T18:15:39,297 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:39,300 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#353 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:39,300 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/76b47eecd646423d95056210c0a582ec is 50, key is test_row_0/B:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:39,333 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112fc556d4746d24d55a692dace2addbd06_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:39,337 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112fc556d4746d24d55a692dace2addbd06_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:39,337 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fc556d4746d24d55a692dace2addbd06_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:39,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T18:15:39,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,348 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742241_1417 (size=12629) 2024-11-12T18:15:39,380 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/76b47eecd646423d95056210c0a582ec as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/76b47eecd646423d95056210c0a582ec 2024-11-12T18:15:39,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742242_1418 (size=4469) 2024-11-12T18:15:39,389 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into 76b47eecd646423d95056210c0a582ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:39,389 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:39,389 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=12, startTime=1731435339254; duration=0sec 2024-11-12T18:15:39,390 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:39,390 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:39,390 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:39,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:39,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:39,391 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:39,391 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/af06de348b854350bf36870014143360, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/76df9ab48693416aa317e629bea88b57, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/985987de29c24f8b9a84ed2e9e4aeeaf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/1754b0aad0154ef4a366d37a2e79cd8e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=47.8 K 2024-11-12T18:15:39,391 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting af06de348b854350bf36870014143360, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731435332764 2024-11-12T18:15:39,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 76df9ab48693416aa317e629bea88b57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731435333501 2024-11-12T18:15:39,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 985987de29c24f8b9a84ed2e9e4aeeaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731435334674 2024-11-12T18:15:39,392 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1754b0aad0154ef4a366d37a2e79cd8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731435337047 2024-11-12T18:15:39,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122446393345de443f83ede725e6824275_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435337749/Put/seqid=0 2024-11-12T18:15:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742243_1419 (size=12304) 2024-11-12T18:15:39,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:39,452 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:39,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122446393345de443f83ede725e6824275_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122446393345de443f83ede725e6824275_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:39,453 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/cb38ac6cdeb3432e92229c947c28006f is 50, key is test_row_0/C:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/72e25c8eadc949c9af76b73b49014818, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:39,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/72e25c8eadc949c9af76b73b49014818 is 175, key is test_row_0/A:col10/1731435337749/Put/seqid=0 2024-11-12T18:15:39,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742244_1420 (size=12629) 2024-11-12T18:15:39,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742245_1421 (size=31105) 2024-11-12T18:15:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T18:15:39,783 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#354 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:39,784 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/120d544af10c4b6f9e81eac318f6cdab is 175, key is test_row_0/A:col10/1731435337721/Put/seqid=0 2024-11-12T18:15:39,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742246_1422 (size=31583) 2024-11-12T18:15:39,798 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/120d544af10c4b6f9e81eac318f6cdab as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/120d544af10c4b6f9e81eac318f6cdab 2024-11-12T18:15:39,807 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into 120d544af10c4b6f9e81eac318f6cdab(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:39,808 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:39,808 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=12, startTime=1731435339253; duration=0sec 2024-11-12T18:15:39,808 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:39,808 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:39,918 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/cb38ac6cdeb3432e92229c947c28006f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/cb38ac6cdeb3432e92229c947c28006f 2024-11-12T18:15:39,923 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into cb38ac6cdeb3432e92229c947c28006f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:39,923 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:39,923 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=12, startTime=1731435339254; duration=0sec 2024-11-12T18:15:39,924 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:39,924 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:39,938 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/72e25c8eadc949c9af76b73b49014818 2024-11-12T18:15:39,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/da76a1667f3f49bc816dfcea7699d3c4 is 50, key is test_row_0/B:col10/1731435337749/Put/seqid=0 2024-11-12T18:15:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742247_1423 (size=12151) 2024-11-12T18:15:39,966 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/da76a1667f3f49bc816dfcea7699d3c4 2024-11-12T18:15:39,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/4869c7d89b8b4f639e36de8bdd7f7f27 is 50, key is test_row_0/C:col10/1731435337749/Put/seqid=0 2024-11-12T18:15:39,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742248_1424 (size=12151) 2024-11-12T18:15:39,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/4869c7d89b8b4f639e36de8bdd7f7f27 2024-11-12T18:15:39,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/72e25c8eadc949c9af76b73b49014818 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818 2024-11-12T18:15:39,998 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818, entries=150, sequenceid=240, filesize=30.4 K 2024-11-12T18:15:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/da76a1667f3f49bc816dfcea7699d3c4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/da76a1667f3f49bc816dfcea7699d3c4 2024-11-12T18:15:40,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/da76a1667f3f49bc816dfcea7699d3c4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-12T18:15:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/4869c7d89b8b4f639e36de8bdd7f7f27 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/4869c7d89b8b4f639e36de8bdd7f7f27 2024-11-12T18:15:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,010 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/4869c7d89b8b4f639e36de8bdd7f7f27, entries=150, sequenceid=240, filesize=11.9 K 2024-11-12T18:15:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,011 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 3946528656bca5567e369f3af446cf65 in 664ms, sequenceid=240, compaction requested=false 2024-11-12T18:15:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:40,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-12T18:15:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-12T18:15:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T18:15:40,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-11-12T18:15:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4590 sec 2024-11-12T18:15:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:15:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:40,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bb7e7ef113d64ae9a79dfb3cc3e98f53_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742249_1425 (size=19774) 2024-11-12T18:15:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:40,248 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bb7e7ef113d64ae9a79dfb3cc3e98f53_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bb7e7ef113d64ae9a79dfb3cc3e98f53_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:40,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7dc4b3665e564234af0b88e222df6016, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:40,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7dc4b3665e564234af0b88e222df6016 is 175, key is test_row_0/A:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:40,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742250_1426 (size=57033) 2024-11-12T18:15:40,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7dc4b3665e564234af0b88e222df6016 2024-11-12T18:15:40,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/09b54d6f097d44c8b5679170a05c031f is 50, key is test_row_0/B:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:40,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742251_1427 (size=12151) 2024-11-12T18:15:40,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435340270, max timestamp=1731435340294Num uniques 859; 2024-11-12T18:15:40,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435400312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435400313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435400315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435400322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435400441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435400441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435400441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435400441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435400650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435400660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435400660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435400660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/09b54d6f097d44c8b5679170a05c031f 2024-11-12T18:15:40,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/95007d915faf4f6285819f5b5ea67c39 is 50, key is test_row_0/C:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:40,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742252_1428 (size=12151) 2024-11-12T18:15:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T18:15:40,710 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:40,711 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-12T18:15:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T18:15:40,713 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:40,713 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:40,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T18:15:40,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T18:15:40,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:40,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:40,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:40,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:40,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:40,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435400971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435400980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435400980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:40,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435400980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T18:15:41,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:41,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T18:15:41,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/95007d915faf4f6285819f5b5ea67c39 2024-11-12T18:15:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7dc4b3665e564234af0b88e222df6016 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016 2024-11-12T18:15:41,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016, entries=300, sequenceid=253, filesize=55.7 K 2024-11-12T18:15:41,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/09b54d6f097d44c8b5679170a05c031f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/09b54d6f097d44c8b5679170a05c031f 2024-11-12T18:15:41,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/09b54d6f097d44c8b5679170a05c031f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-12T18:15:41,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/95007d915faf4f6285819f5b5ea67c39 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/95007d915faf4f6285819f5b5ea67c39 2024-11-12T18:15:41,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/95007d915faf4f6285819f5b5ea67c39, entries=150, sequenceid=253, filesize=11.9 K 2024-11-12T18:15:41,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 3946528656bca5567e369f3af446cf65 in 901ms, sequenceid=253, compaction requested=true 2024-11-12T18:15:41,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:41,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:41,116 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:41,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:41,116 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:41,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:41,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:41,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:41,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:41,117 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:41,117 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:41,117 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,118 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/120d544af10c4b6f9e81eac318f6cdab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=116.9 K 2024-11-12T18:15:41,118 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,118 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/120d544af10c4b6f9e81eac318f6cdab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016] 2024-11-12T18:15:41,118 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:41,118 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:41,118 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,118 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/76b47eecd646423d95056210c0a582ec, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/da76a1667f3f49bc816dfcea7699d3c4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/09b54d6f097d44c8b5679170a05c031f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=36.1 K 2024-11-12T18:15:41,118 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120d544af10c4b6f9e81eac318f6cdab, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731435337047 2024-11-12T18:15:41,119 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 76b47eecd646423d95056210c0a582ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731435337047 2024-11-12T18:15:41,119 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e25c8eadc949c9af76b73b49014818, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731435337749 2024-11-12T18:15:41,119 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting da76a1667f3f49bc816dfcea7699d3c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731435337749 2024-11-12T18:15:41,119 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc4b3665e564234af0b88e222df6016, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731435340141 2024-11-12T18:15:41,119 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b54d6f097d44c8b5679170a05c031f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731435340156 2024-11-12T18:15:41,127 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:41,129 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411128b0a528b0d5b41bba86725783372481b_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:41,130 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:41,130 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/006d3b64d8c247c9bd0076312dba8b6c is 50, key is test_row_0/B:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:41,133 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411128b0a528b0d5b41bba86725783372481b_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:41,133 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128b0a528b0d5b41bba86725783372481b_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:41,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742254_1430 (size=4469) 2024-11-12T18:15:41,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742253_1429 (size=12731) 2024-11-12T18:15:41,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=131.94 KB heapSize=386.72 KB 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121cd2ecb1c8cf4490a285a9502c2526f5_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435340311/Put/seqid=0 2024-11-12T18:15:41,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742255_1431 (size=12454) 2024-11-12T18:15:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:41,245 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121cd2ecb1c8cf4490a285a9502c2526f5_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121cd2ecb1c8cf4490a285a9502c2526f5_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:41,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7d1ef89cc892404a87846686daec9c5e, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:41,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7d1ef89cc892404a87846686daec9c5e is 175, key is test_row_0/A:col10/1731435340311/Put/seqid=0 2024-11-12T18:15:41,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742256_1432 (size=31255) 2024-11-12T18:15:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T18:15:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:41,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435401524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435401524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,548 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#362 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:41,549 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/040d1663a5d14fe69fd1d970418c4761 is 175, key is test_row_0/A:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:41,553 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/006d3b64d8c247c9bd0076312dba8b6c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/006d3b64d8c247c9bd0076312dba8b6c 2024-11-12T18:15:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435401534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435401535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742257_1433 (size=31685) 2024-11-12T18:15:41,560 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into 006d3b64d8c247c9bd0076312dba8b6c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:41,560 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:41,560 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=13, startTime=1731435341116; duration=0sec 2024-11-12T18:15:41,560 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:41,560 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:41,561 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:41,563 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:41,563 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:41,563 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:41,563 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/cb38ac6cdeb3432e92229c947c28006f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/4869c7d89b8b4f639e36de8bdd7f7f27, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/95007d915faf4f6285819f5b5ea67c39] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=36.1 K 2024-11-12T18:15:41,564 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting cb38ac6cdeb3432e92229c947c28006f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731435337047 2024-11-12T18:15:41,564 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4869c7d89b8b4f639e36de8bdd7f7f27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731435337749 2024-11-12T18:15:41,564 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 95007d915faf4f6285819f5b5ea67c39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731435340156 2024-11-12T18:15:41,570 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:41,571 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/a0ef1f3830d845b28fb39c24a812e542 is 50, key is test_row_0/C:col10/1731435340212/Put/seqid=0 2024-11-12T18:15:41,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742258_1434 (size=12731) 2024-11-12T18:15:41,582 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/a0ef1f3830d845b28fb39c24a812e542 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a0ef1f3830d845b28fb39c24a812e542 2024-11-12T18:15:41,591 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into a0ef1f3830d845b28fb39c24a812e542(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:41,591 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:41,591 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=13, startTime=1731435341117; duration=0sec 2024-11-12T18:15:41,591 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:41,591 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:41,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435401640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435401643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,655 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7d1ef89cc892404a87846686daec9c5e 2024-11-12T18:15:41,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/552e764bb5f04f9da955e1b706905777 is 50, key is test_row_0/B:col10/1731435340311/Put/seqid=0 2024-11-12T18:15:41,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435401661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435401661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742259_1435 (size=12301) 2024-11-12T18:15:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T18:15:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435401860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435401860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435401881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435401881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:41,970 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/040d1663a5d14fe69fd1d970418c4761 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/040d1663a5d14fe69fd1d970418c4761 2024-11-12T18:15:41,976 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into 040d1663a5d14fe69fd1d970418c4761(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:41,976 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:41,976 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=13, startTime=1731435341116; duration=0sec 2024-11-12T18:15:41,976 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:41,976 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:42,126 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/552e764bb5f04f9da955e1b706905777 2024-11-12T18:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/e5061360aed0407b9dcab9a8f03439ea is 50, key is test_row_0/C:col10/1731435340311/Put/seqid=0 2024-11-12T18:15:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742260_1436 (size=12301) 2024-11-12T18:15:42,156 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/e5061360aed0407b9dcab9a8f03439ea 2024-11-12T18:15:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7d1ef89cc892404a87846686daec9c5e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e 2024-11-12T18:15:42,168 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e, entries=150, sequenceid=280, filesize=30.5 K 2024-11-12T18:15:42,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/552e764bb5f04f9da955e1b706905777 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/552e764bb5f04f9da955e1b706905777 2024-11-12T18:15:42,173 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/552e764bb5f04f9da955e1b706905777, entries=150, sequenceid=280, filesize=12.0 K 2024-11-12T18:15:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/e5061360aed0407b9dcab9a8f03439ea as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e5061360aed0407b9dcab9a8f03439ea 2024-11-12T18:15:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e5061360aed0407b9dcab9a8f03439ea, entries=150, sequenceid=280, filesize=12.0 K 2024-11-12T18:15:42,181 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~386.67 KB/395952, currentSize=46.96 KB/48090 for 3946528656bca5567e369f3af446cf65 in 1006ms, sequenceid=280, compaction requested=false 2024-11-12T18:15:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-12T18:15:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-12T18:15:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:42,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:42,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:42,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:42,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-12T18:15:42,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-12T18:15:42,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4820 sec 2024-11-12T18:15:42,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112faeed73101c84493857701a19bb084fa_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:42,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742261_1437 (size=12454) 2024-11-12T18:15:42,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435342251, max timestamp=1731435342286Num uniques 859; 2024-11-12T18:15:42,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:42,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435342251, max timestamp=1731435342286Num uniques 859; 2024-11-12T18:15:42,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:42,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435342251, max timestamp=1731435342286Num uniques 859; 2024-11-12T18:15:42,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:15:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435402356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435402357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435402376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435402377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435402480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435402480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435402480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435402500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:42,616 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112faeed73101c84493857701a19bb084fa_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112faeed73101c84493857701a19bb084fa_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:42,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/f2128c9113e14cbca0f05858220c5962, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:42,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/f2128c9113e14cbca0f05858220c5962 is 175, key is test_row_0/A:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742262_1438 (size=31255) 2024-11-12T18:15:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435402701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435402701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435402701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435402720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T18:15:42,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:42,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-12T18:15:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T18:15:42,864 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:42,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:42,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T18:15:43,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T18:15:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:43,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435403010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/f2128c9113e14cbca0f05858220c5962 2024-11-12T18:15:43,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8b24e4a5171949f2a2a3cef3da7a909e is 50, key is test_row_0/B:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435403020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435403020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742263_1439 (size=12301) 2024-11-12T18:15:43,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8b24e4a5171949f2a2a3cef3da7a909e 2024-11-12T18:15:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435403041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6457cb893d664be4adfdb3bc82c967c3 is 50, key is test_row_0/C:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:43,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742264_1440 (size=12301) 2024-11-12T18:15:43,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6457cb893d664be4adfdb3bc82c967c3 2024-11-12T18:15:43,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/f2128c9113e14cbca0f05858220c5962 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962 2024-11-12T18:15:43,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962, entries=150, sequenceid=294, filesize=30.5 K 2024-11-12T18:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8b24e4a5171949f2a2a3cef3da7a909e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8b24e4a5171949f2a2a3cef3da7a909e 2024-11-12T18:15:43,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8b24e4a5171949f2a2a3cef3da7a909e, entries=150, sequenceid=294, filesize=12.0 K 2024-11-12T18:15:43,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/6457cb893d664be4adfdb3bc82c967c3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6457cb893d664be4adfdb3bc82c967c3 2024-11-12T18:15:43,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6457cb893d664be4adfdb3bc82c967c3, entries=150, sequenceid=294, filesize=12.0 K 2024-11-12T18:15:43,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3946528656bca5567e369f3af446cf65 in 904ms, sequenceid=294, compaction requested=true 2024-11-12T18:15:43,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:43,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:43,092 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:43,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:43,092 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:43,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:43,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:43,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:43,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:43,096 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:43,096 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:43,096 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:43,096 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:43,096 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,096 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,096 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/040d1663a5d14fe69fd1d970418c4761, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=92.0 K 2024-11-12T18:15:43,096 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/006d3b64d8c247c9bd0076312dba8b6c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/552e764bb5f04f9da955e1b706905777, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8b24e4a5171949f2a2a3cef3da7a909e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=36.5 K 2024-11-12T18:15:43,096 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,096 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/040d1663a5d14fe69fd1d970418c4761, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962] 2024-11-12T18:15:43,096 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 006d3b64d8c247c9bd0076312dba8b6c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731435340156 2024-11-12T18:15:43,097 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040d1663a5d14fe69fd1d970418c4761, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731435340156 2024-11-12T18:15:43,097 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 552e764bb5f04f9da955e1b706905777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731435340303 2024-11-12T18:15:43,097 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b24e4a5171949f2a2a3cef3da7a909e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731435341515 2024-11-12T18:15:43,097 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d1ef89cc892404a87846686daec9c5e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731435340303 2024-11-12T18:15:43,097 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2128c9113e14cbca0f05858220c5962, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731435341515 2024-11-12T18:15:43,103 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:43,105 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:43,106 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/5d6527fc237441628f983c0b95cfe092 is 50, key is test_row_0/B:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:43,107 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411125ba1e0cb034e4f6383e7c4774b972ea8_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:43,109 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411125ba1e0cb034e4f6383e7c4774b972ea8_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:43,109 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125ba1e0cb034e4f6383e7c4774b972ea8_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:43,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742266_1442 (size=4469) 2024-11-12T18:15:43,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742265_1441 (size=12983) 2024-11-12T18:15:43,125 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#372 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:43,126 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/adc8823b48ab40cfa3b98b3d38064d6c is 175, key is test_row_0/A:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:43,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742267_1443 (size=31937) 2024-11-12T18:15:43,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T18:15:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T18:15:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T18:15:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111268d4484ac2a348c2bff0bb4e4b0d0ff9_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435342374/Put/seqid=0 2024-11-12T18:15:43,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742268_1444 (size=12454) 2024-11-12T18:15:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T18:15:43,527 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/5d6527fc237441628f983c0b95cfe092 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/5d6527fc237441628f983c0b95cfe092 2024-11-12T18:15:43,532 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into 5d6527fc237441628f983c0b95cfe092(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:43,532 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:43,532 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=13, startTime=1731435343092; duration=0sec 2024-11-12T18:15:43,536 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:43,536 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:43,537 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:43,537 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:43,537 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:43,537 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:43,537 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a0ef1f3830d845b28fb39c24a812e542, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e5061360aed0407b9dcab9a8f03439ea, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6457cb893d664be4adfdb3bc82c967c3] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=36.5 K 2024-11-12T18:15:43,538 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ef1f3830d845b28fb39c24a812e542, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731435340156 2024-11-12T18:15:43,539 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/adc8823b48ab40cfa3b98b3d38064d6c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/adc8823b48ab40cfa3b98b3d38064d6c 2024-11-12T18:15:43,539 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e5061360aed0407b9dcab9a8f03439ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731435340303 2024-11-12T18:15:43,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6457cb893d664be4adfdb3bc82c967c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731435341515 2024-11-12T18:15:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:43,550 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into adc8823b48ab40cfa3b98b3d38064d6c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:43,550 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:43,550 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=13, startTime=1731435343091; duration=0sec 2024-11-12T18:15:43,550 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:43,550 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:43,555 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:43,555 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/84e4925bfd8e4703bc3b5297069f50ce is 50, key is test_row_0/C:col10/1731435341533/Put/seqid=0 2024-11-12T18:15:43,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742269_1445 (size=12983) 2024-11-12T18:15:43,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435403571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435403572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435403574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435403574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:43,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111268d4484ac2a348c2bff0bb4e4b0d0ff9_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111268d4484ac2a348c2bff0bb4e4b0d0ff9_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c8fca0019bbb42a4a494b1832061f35d, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c8fca0019bbb42a4a494b1832061f35d is 175, key is test_row_0/A:col10/1731435342374/Put/seqid=0 2024-11-12T18:15:43,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742270_1446 (size=31255) 2024-11-12T18:15:43,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435403680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435403690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435403690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435403900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435403900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435403900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:43,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:43,966 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/84e4925bfd8e4703bc3b5297069f50ce as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/84e4925bfd8e4703bc3b5297069f50ce 2024-11-12T18:15:43,971 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into 84e4925bfd8e4703bc3b5297069f50ce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:43,971 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:43,971 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=13, startTime=1731435343092; duration=0sec 2024-11-12T18:15:43,971 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:43,971 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T18:15:44,047 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c8fca0019bbb42a4a494b1832061f35d 2024-11-12T18:15:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/d428f528ae1148e3895737b76e103629 is 50, key is test_row_0/B:col10/1731435342374/Put/seqid=0 2024-11-12T18:15:44,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742271_1447 (size=12301) 2024-11-12T18:15:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435404221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435404221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435404221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/d428f528ae1148e3895737b76e103629 2024-11-12T18:15:44,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/81ecadf7313244c7ba4a42e5f57e665f is 50, key is test_row_0/C:col10/1731435342374/Put/seqid=0 2024-11-12T18:15:44,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742272_1448 (size=12301) 2024-11-12T18:15:44,480 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/81ecadf7313244c7ba4a42e5f57e665f 2024-11-12T18:15:44,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c8fca0019bbb42a4a494b1832061f35d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d 2024-11-12T18:15:44,509 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d, entries=150, sequenceid=331, filesize=30.5 K 2024-11-12T18:15:44,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/d428f528ae1148e3895737b76e103629 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/d428f528ae1148e3895737b76e103629 2024-11-12T18:15:44,519 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/d428f528ae1148e3895737b76e103629, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T18:15:44,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/81ecadf7313244c7ba4a42e5f57e665f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/81ecadf7313244c7ba4a42e5f57e665f 2024-11-12T18:15:44,526 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/81ecadf7313244c7ba4a42e5f57e665f, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T18:15:44,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3946528656bca5567e369f3af446cf65 in 1350ms, sequenceid=331, compaction requested=false 2024-11-12T18:15:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-12T18:15:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-12T18:15:44,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T18:15:44,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-12T18:15:44,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.6700 sec 2024-11-12T18:15:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:44,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:44,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:44,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:44,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:44,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:44,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:44,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:44,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121f4e2ea1778e47998597f6513f571096_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:44,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742273_1449 (size=12454) 2024-11-12T18:15:44,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435404810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435404812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435344779, max timestamp=1731435344830Num uniques 859; 2024-11-12T18:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435344779, max timestamp=1731435344830Num uniques 859; 2024-11-12T18:15:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:44,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435344779, max timestamp=1731435344830Num uniques 859; 2024-11-12T18:15:44,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435404940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435404940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435404940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:44,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435404944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T18:15:45,021 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:45,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-12T18:15:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:45,033 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:45,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:45,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435405060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,078 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:45,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435405073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121f4e2ea1778e47998597f6513f571096_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121f4e2ea1778e47998597f6513f571096_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:45,092 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/79e02a3a20964faeaf373026a3cc5475, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:45,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/79e02a3a20964faeaf373026a3cc5475 is 175, key is test_row_0/A:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:45,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742274_1450 (size=31255) 2024-11-12T18:15:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435405172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435405175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435405272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435405300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435405480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435405490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/79e02a3a20964faeaf373026a3cc5475 2024-11-12T18:15:45,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/774493d0b1ca47c79506a4112831b0cb is 50, key is test_row_0/B:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:45,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742275_1451 (size=12301) 2024-11-12T18:15:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435405597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435405625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:45,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:45,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:45,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:45,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:45,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:45,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:45,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/774493d0b1ca47c79506a4112831b0cb 2024-11-12T18:15:46,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35036 deadline: 1731435406000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:46,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/5cf201df8fa64a1aa1fc639f3418ba20 is 50, key is test_row_0/C:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:46,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34948 deadline: 1731435406013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:46,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742276_1452 (size=12301) 2024-11-12T18:15:46,054 INFO [Thread-1642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1642. 2024-11-12T18:15:46,054 DEBUG [Thread-1642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:46,054 DEBUG [Thread-1642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,054 DEBUG [Thread-1642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,059 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-11-12T18:15:46,059 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:46,059 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,059 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:46,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:46,074 INFO [Thread-1646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1646. 2024-11-12T18:15:46,074 DEBUG [Thread-1646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:46,074 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,074 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,075 INFO [Thread-1644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1644. 2024-11-12T18:15:46,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:46,075 DEBUG [Thread-1644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:46,075 DEBUG [Thread-1644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,075 DEBUG [Thread-1644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:46,088 INFO [Thread-1648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1648. 2024-11-12T18:15:46,088 DEBUG [Thread-1648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:46,088 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,088 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:46,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:46,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34956 deadline: 1731435406141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:46,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34974 deadline: 1731435406151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:15:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:46,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/5cf201df8fa64a1aa1fc639f3418ba20 2024-11-12T18:15:46,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/79e02a3a20964faeaf373026a3cc5475 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475 2024-11-12T18:15:46,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475, entries=150, sequenceid=345, filesize=30.5 K 2024-11-12T18:15:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/774493d0b1ca47c79506a4112831b0cb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/774493d0b1ca47c79506a4112831b0cb 2024-11-12T18:15:46,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:46,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:46,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/774493d0b1ca47c79506a4112831b0cb, entries=150, sequenceid=345, filesize=12.0 K 2024-11-12T18:15:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:46,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/5cf201df8fa64a1aa1fc639f3418ba20 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/5cf201df8fa64a1aa1fc639f3418ba20 2024-11-12T18:15:46,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/5cf201df8fa64a1aa1fc639f3418ba20, entries=150, sequenceid=345, filesize=12.0 K 2024-11-12T18:15:46,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3946528656bca5567e369f3af446cf65 in 1875ms, sequenceid=345, compaction requested=true 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3946528656bca5567e369f3af446cf65:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:46,461 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:46,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:46,461 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:46,461 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:46,462 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/B is initiating minor compaction (all files) 2024-11-12T18:15:46,462 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/A is initiating minor compaction (all files) 2024-11-12T18:15:46,462 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/B in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,462 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/A in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,462 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/5d6527fc237441628f983c0b95cfe092, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/d428f528ae1148e3895737b76e103629, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/774493d0b1ca47c79506a4112831b0cb] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=36.7 K 2024-11-12T18:15:46,462 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/adc8823b48ab40cfa3b98b3d38064d6c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=92.2 K 2024-11-12T18:15:46,462 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,462 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/adc8823b48ab40cfa3b98b3d38064d6c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475] 2024-11-12T18:15:46,462 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d6527fc237441628f983c0b95cfe092, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731435341515 2024-11-12T18:15:46,462 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc8823b48ab40cfa3b98b3d38064d6c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731435341515 2024-11-12T18:15:46,463 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d428f528ae1148e3895737b76e103629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731435342351 2024-11-12T18:15:46,463 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8fca0019bbb42a4a494b1832061f35d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731435342351 2024-11-12T18:15:46,463 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e02a3a20964faeaf373026a3cc5475, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731435343571 2024-11-12T18:15:46,463 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 774493d0b1ca47c79506a4112831b0cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731435343571 2024-11-12T18:15:46,474 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#B#compaction#380 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:46,474 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8d55b5c6619349beb93034e526684c7e is 50, key is test_row_0/B:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:46,475 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:46,483 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112aa861b3e14a64f61953a397c5b10ccb9_3946528656bca5567e369f3af446cf65 store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:46,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742277_1453 (size=13085) 2024-11-12T18:15:46,510 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112aa861b3e14a64f61953a397c5b10ccb9_3946528656bca5567e369f3af446cf65, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:46,510 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112aa861b3e14a64f61953a397c5b10ccb9_3946528656bca5567e369f3af446cf65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:46,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742278_1454 (size=4469) 2024-11-12T18:15:46,543 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#A#compaction#381 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:46,544 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/b5a497e92e37404cb1f8ab9d65024d8e is 175, key is test_row_0/A:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:46,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742279_1455 (size=32039) 2024-11-12T18:15:46,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T18:15:46,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,610 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-12T18:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:46,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124c1caa8737aa49d9b5f6bf0cf2da21eb_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435344939/Put/seqid=0 2024-11-12T18:15:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742280_1456 (size=12454) 2024-11-12T18:15:46,892 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/8d55b5c6619349beb93034e526684c7e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8d55b5c6619349beb93034e526684c7e 2024-11-12T18:15:46,896 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/B of 3946528656bca5567e369f3af446cf65 into 8d55b5c6619349beb93034e526684c7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:46,896 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:46,896 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/B, priority=13, startTime=1731435346461; duration=0sec 2024-11-12T18:15:46,896 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:46,896 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:B 2024-11-12T18:15:46,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:15:46,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:15:46,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 3946528656bca5567e369f3af446cf65/C is initiating minor compaction (all files) 2024-11-12T18:15:46,898 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3946528656bca5567e369f3af446cf65/C in TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:46,898 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/84e4925bfd8e4703bc3b5297069f50ce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/81ecadf7313244c7ba4a42e5f57e665f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/5cf201df8fa64a1aa1fc639f3418ba20] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp, totalSize=36.7 K 2024-11-12T18:15:46,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e4925bfd8e4703bc3b5297069f50ce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731435341515 2024-11-12T18:15:46,898 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ecadf7313244c7ba4a42e5f57e665f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731435342351 2024-11-12T18:15:46,899 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf201df8fa64a1aa1fc639f3418ba20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731435343571 2024-11-12T18:15:46,907 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3946528656bca5567e369f3af446cf65#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:46,908 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/52243141ddfe4360836e73d2d22b4111 is 50, key is test_row_0/C:col10/1731435343572/Put/seqid=0 2024-11-12T18:15:46,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742281_1457 (size=13085) 2024-11-12T18:15:46,974 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/b5a497e92e37404cb1f8ab9d65024d8e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/b5a497e92e37404cb1f8ab9d65024d8e 2024-11-12T18:15:46,978 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/A of 3946528656bca5567e369f3af446cf65 into b5a497e92e37404cb1f8ab9d65024d8e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:46,978 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:46,978 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/A, priority=13, startTime=1731435346461; duration=0sec 2024-11-12T18:15:46,978 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:46,978 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:A 2024-11-12T18:15:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:47,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. as already flushing 2024-11-12T18:15:47,032 INFO [Thread-1633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1633. 2024-11-12T18:15:47,032 DEBUG [Thread-1633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:47,032 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,032 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:47,052 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-11-12T18:15:47,052 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:47,052 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,053 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:47,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:47,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124c1caa8737aa49d9b5f6bf0cf2da21eb_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124c1caa8737aa49d9b5f6bf0cf2da21eb_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:47,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c186da54b82347d69f52e478c28013c9, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:47,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c186da54b82347d69f52e478c28013c9 is 175, key is test_row_0/A:col10/1731435344939/Put/seqid=0 2024-11-12T18:15:47,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742282_1458 (size=31255) 2024-11-12T18:15:47,162 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-12T18:15:47,163 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:47,163 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,163 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:47,172 INFO [Thread-1639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1639. 2024-11-12T18:15:47,172 DEBUG [Thread-1639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:47,172 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,172 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:47,317 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/52243141ddfe4360836e73d2d22b4111 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/52243141ddfe4360836e73d2d22b4111 2024-11-12T18:15:47,321 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3946528656bca5567e369f3af446cf65/C of 3946528656bca5567e369f3af446cf65 into 52243141ddfe4360836e73d2d22b4111(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:47,321 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:47,321 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., storeName=3946528656bca5567e369f3af446cf65/C, priority=13, startTime=1731435346461; duration=0sec 2024-11-12T18:15:47,321 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:47,321 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3946528656bca5567e369f3af446cf65:C 2024-11-12T18:15:47,495 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c186da54b82347d69f52e478c28013c9 2024-11-12T18:15:47,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/0d47b6d0be2f4e54ad2f9fed4a831fb7 is 50, key is test_row_0/B:col10/1731435344939/Put/seqid=0 2024-11-12T18:15:47,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742283_1459 (size=12301) 2024-11-12T18:15:47,906 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/0d47b6d0be2f4e54ad2f9fed4a831fb7 2024-11-12T18:15:47,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/66cd72b3c27747cdaacea4d4204bdfdb is 50, key is test_row_0/C:col10/1731435344939/Put/seqid=0 2024-11-12T18:15:47,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742284_1460 (size=12301) 2024-11-12T18:15:47,972 INFO [Thread-1637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1637. 2024-11-12T18:15:47,972 DEBUG [Thread-1637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:15:47,972 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,972 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:47,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:48,320 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/66cd72b3c27747cdaacea4d4204bdfdb 2024-11-12T18:15:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/c186da54b82347d69f52e478c28013c9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c186da54b82347d69f52e478c28013c9 2024-11-12T18:15:48,327 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c186da54b82347d69f52e478c28013c9, entries=150, sequenceid=380, filesize=30.5 K 2024-11-12T18:15:48,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/0d47b6d0be2f4e54ad2f9fed4a831fb7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0d47b6d0be2f4e54ad2f9fed4a831fb7 2024-11-12T18:15:48,331 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0d47b6d0be2f4e54ad2f9fed4a831fb7, entries=150, sequenceid=380, filesize=12.0 K 2024-11-12T18:15:48,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/66cd72b3c27747cdaacea4d4204bdfdb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/66cd72b3c27747cdaacea4d4204bdfdb 2024-11-12T18:15:48,334 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/66cd72b3c27747cdaacea4d4204bdfdb, entries=150, sequenceid=380, filesize=12.0 K 2024-11-12T18:15:48,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=33.54 KB/34350 for 3946528656bca5567e369f3af446cf65 in 1725ms, sequenceid=380, compaction requested=false 2024-11-12T18:15:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 3946528656bca5567e369f3af446cf65: 2024-11-12T18:15:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-12T18:15:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-12T18:15:48,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T18:15:48,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3000 sec 2024-11-12T18:15:48,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.3130 sec 2024-11-12T18:15:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T18:15:49,211 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 776 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2328 rows 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 779 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2336 rows 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 774 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2322 rows 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 770 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2309 rows 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 775 2024-11-12T18:15:49,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2325 rows 2024-11-12T18:15:49,212 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:15:49,212 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:15:49,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:49,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:49,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:15:49,214 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T18:15:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T18:15:49,218 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435349218"}]},"ts":"1731435349218"} 2024-11-12T18:15:49,219 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T18:15:49,219 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T18:15:49,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:15:49,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, UNASSIGN}] 2024-11-12T18:15:49,222 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, UNASSIGN 2024-11-12T18:15:49,223 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:49,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, UNASSIGN because future has completed 2024-11-12T18:15:49,224 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:15:49,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:15:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T18:15:49,377 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:49,377 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:15:49,377 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 3946528656bca5567e369f3af446cf65, disabling compactions & flushes 2024-11-12T18:15:49,377 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:49,377 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:49,377 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. after waiting 0 ms 2024-11-12T18:15:49,377 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:49,377 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 3946528656bca5567e369f3af446cf65 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T18:15:49,378 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=A 2024-11-12T18:15:49,378 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:49,378 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=B 2024-11-12T18:15:49,378 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:49,378 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3946528656bca5567e369f3af446cf65, store=C 2024-11-12T18:15:49,378 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:49,383 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112904632d4e2e845629c2bbcd9336ef2e5_3946528656bca5567e369f3af446cf65 is 50, key is test_row_0/A:col10/1731435347971/Put/seqid=0 2024-11-12T18:15:49,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742285_1461 (size=12454) 2024-11-12T18:15:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T18:15:49,787 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:49,791 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112904632d4e2e845629c2bbcd9336ef2e5_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112904632d4e2e845629c2bbcd9336ef2e5_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:49,791 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c68e21ac7124922b5fc28372f366c47, store: [table=TestAcidGuarantees family=A region=3946528656bca5567e369f3af446cf65] 2024-11-12T18:15:49,792 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c68e21ac7124922b5fc28372f366c47 is 175, key is test_row_0/A:col10/1731435347971/Put/seqid=0 2024-11-12T18:15:49,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742286_1462 (size=31255) 2024-11-12T18:15:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T18:15:50,196 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c68e21ac7124922b5fc28372f366c47 2024-11-12T18:15:50,202 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/fa0eea92220643519049deda8b0bab89 is 50, key is test_row_0/B:col10/1731435347971/Put/seqid=0 2024-11-12T18:15:50,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742287_1463 (size=12301) 2024-11-12T18:15:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T18:15:50,608 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/fa0eea92220643519049deda8b0bab89 2024-11-12T18:15:50,614 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/c36a82bd60cb407a9dc6059a4fb77927 is 50, key is test_row_0/C:col10/1731435347971/Put/seqid=0 2024-11-12T18:15:50,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742288_1464 (size=12301) 2024-11-12T18:15:51,018 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/c36a82bd60cb407a9dc6059a4fb77927 2024-11-12T18:15:51,022 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/A/7c68e21ac7124922b5fc28372f366c47 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c68e21ac7124922b5fc28372f366c47 2024-11-12T18:15:51,026 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c68e21ac7124922b5fc28372f366c47, entries=150, sequenceid=391, filesize=30.5 K 2024-11-12T18:15:51,027 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/B/fa0eea92220643519049deda8b0bab89 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/fa0eea92220643519049deda8b0bab89 2024-11-12T18:15:51,031 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/fa0eea92220643519049deda8b0bab89, entries=150, sequenceid=391, filesize=12.0 K 2024-11-12T18:15:51,031 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/.tmp/C/c36a82bd60cb407a9dc6059a4fb77927 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/c36a82bd60cb407a9dc6059a4fb77927 2024-11-12T18:15:51,035 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/c36a82bd60cb407a9dc6059a4fb77927, entries=150, sequenceid=391, filesize=12.0 K 2024-11-12T18:15:51,036 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3946528656bca5567e369f3af446cf65 in 1659ms, sequenceid=391, compaction requested=true 2024-11-12T18:15:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/8297c6b16c554ee0a3d3f5d1c42c9a2d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ea53a5942bab4be6b06fb25857341a33, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/9b06a5736414461fb411c5a2b1756a0a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/120d544af10c4b6f9e81eac318f6cdab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/040d1663a5d14fe69fd1d970418c4761, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/adc8823b48ab40cfa3b98b3d38064d6c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475] to archive 2024-11-12T18:15:51,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:15:51,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3c40ae7b22f240e9916ec36a88caa165 2024-11-12T18:15:51,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c31624e3dfb47afa028c40584b47739 2024-11-12T18:15:51,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/8297c6b16c554ee0a3d3f5d1c42c9a2d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/8297c6b16c554ee0a3d3f5d1c42c9a2d 2024-11-12T18:15:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6ef74fef4ad4436ba11522eb34489ad 2024-11-12T18:15:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/88cab1145e0c46e4892e3061345c10e7 2024-11-12T18:15:51,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2bf68a5e99644d4a8c9ad17b1b5140da 2024-11-12T18:15:51,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c9a9be3882574e28b8c6172c5f231c71 2024-11-12T18:15:51,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ea53a5942bab4be6b06fb25857341a33 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ea53a5942bab4be6b06fb25857341a33 2024-11-12T18:15:51,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/2a82627d38854d8e8313bb1372e4d852 2024-11-12T18:15:51,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/9b06a5736414461fb411c5a2b1756a0a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/9b06a5736414461fb411c5a2b1756a0a 2024-11-12T18:15:51,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/805cc704cf624e01a337e4dae407a399 2024-11-12T18:15:51,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/e6f62dc41cc64745a05ff45f955641d3 2024-11-12T18:15:51,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/ae8995c09f744633aa49fa7708b60548 2024-11-12T18:15:51,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/3a15be8de2014b8eb9a1ddbcd145905e 2024-11-12T18:15:51,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/120d544af10c4b6f9e81eac318f6cdab to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/120d544af10c4b6f9e81eac318f6cdab 2024-11-12T18:15:51,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/72e25c8eadc949c9af76b73b49014818 2024-11-12T18:15:51,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7dc4b3665e564234af0b88e222df6016 2024-11-12T18:15:51,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/040d1663a5d14fe69fd1d970418c4761 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/040d1663a5d14fe69fd1d970418c4761 2024-11-12T18:15:51,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7d1ef89cc892404a87846686daec9c5e 2024-11-12T18:15:51,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/adc8823b48ab40cfa3b98b3d38064d6c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/adc8823b48ab40cfa3b98b3d38064d6c 2024-11-12T18:15:51,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/f2128c9113e14cbca0f05858220c5962 2024-11-12T18:15:51,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c8fca0019bbb42a4a494b1832061f35d 2024-11-12T18:15:51,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/79e02a3a20964faeaf373026a3cc5475 2024-11-12T18:15:51,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0ae4c6a41c8d4cfd955e518e36914d8d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/7187aff9c49643fa8a395995ec9dca96, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/bd9678cc6bcb4463a969740633c18c29, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/12bd03202bc6411297c25110375d42cb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/61a7a03d5e8b4e0ab07fa36bbd1df64d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/47e12ae99e774053862ccdb6192891b4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/15b30e1a1c9547a6896ebbf505f97f8a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/6bf96a42055e420988f265aec31d08c8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/85893ea7c6634bd18b1a85311011c194, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/2beb238e30d149a3a80e6686f53b8021, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/897f9c18cc554d338aa13974d8159e59, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8be811ea0bc246fb8aa3c5cb8e09a3fe, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/aa70ed4792574f1eb33ad9c21866ab43, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/76b47eecd646423d95056210c0a582ec, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/c7708e8e75f44060a040a2f720ab7f85, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/da76a1667f3f49bc816dfcea7699d3c4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/006d3b64d8c247c9bd0076312dba8b6c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/09b54d6f097d44c8b5679170a05c031f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/552e764bb5f04f9da955e1b706905777, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/5d6527fc237441628f983c0b95cfe092, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8b24e4a5171949f2a2a3cef3da7a909e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/d428f528ae1148e3895737b76e103629, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/774493d0b1ca47c79506a4112831b0cb] to archive 2024-11-12T18:15:51,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:15:51,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0ae4c6a41c8d4cfd955e518e36914d8d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0ae4c6a41c8d4cfd955e518e36914d8d 2024-11-12T18:15:51,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/7187aff9c49643fa8a395995ec9dca96 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/7187aff9c49643fa8a395995ec9dca96 2024-11-12T18:15:51,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/bd9678cc6bcb4463a969740633c18c29 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/bd9678cc6bcb4463a969740633c18c29 2024-11-12T18:15:51,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/12bd03202bc6411297c25110375d42cb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/12bd03202bc6411297c25110375d42cb 2024-11-12T18:15:51,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/61a7a03d5e8b4e0ab07fa36bbd1df64d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/61a7a03d5e8b4e0ab07fa36bbd1df64d 2024-11-12T18:15:51,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/47e12ae99e774053862ccdb6192891b4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/47e12ae99e774053862ccdb6192891b4 2024-11-12T18:15:51,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/15b30e1a1c9547a6896ebbf505f97f8a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/15b30e1a1c9547a6896ebbf505f97f8a 2024-11-12T18:15:51,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/6bf96a42055e420988f265aec31d08c8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/6bf96a42055e420988f265aec31d08c8 2024-11-12T18:15:51,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/85893ea7c6634bd18b1a85311011c194 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/85893ea7c6634bd18b1a85311011c194 2024-11-12T18:15:51,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/2beb238e30d149a3a80e6686f53b8021 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/2beb238e30d149a3a80e6686f53b8021 2024-11-12T18:15:51,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/897f9c18cc554d338aa13974d8159e59 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/897f9c18cc554d338aa13974d8159e59 2024-11-12T18:15:51,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8be811ea0bc246fb8aa3c5cb8e09a3fe to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8be811ea0bc246fb8aa3c5cb8e09a3fe 2024-11-12T18:15:51,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/aa70ed4792574f1eb33ad9c21866ab43 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/aa70ed4792574f1eb33ad9c21866ab43 2024-11-12T18:15:51,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/76b47eecd646423d95056210c0a582ec to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/76b47eecd646423d95056210c0a582ec 2024-11-12T18:15:51,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/c7708e8e75f44060a040a2f720ab7f85 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/c7708e8e75f44060a040a2f720ab7f85 2024-11-12T18:15:51,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/da76a1667f3f49bc816dfcea7699d3c4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/da76a1667f3f49bc816dfcea7699d3c4 2024-11-12T18:15:51,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/006d3b64d8c247c9bd0076312dba8b6c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/006d3b64d8c247c9bd0076312dba8b6c 2024-11-12T18:15:51,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/09b54d6f097d44c8b5679170a05c031f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/09b54d6f097d44c8b5679170a05c031f 2024-11-12T18:15:51,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/552e764bb5f04f9da955e1b706905777 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/552e764bb5f04f9da955e1b706905777 2024-11-12T18:15:51,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/5d6527fc237441628f983c0b95cfe092 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/5d6527fc237441628f983c0b95cfe092 2024-11-12T18:15:51,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8b24e4a5171949f2a2a3cef3da7a909e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8b24e4a5171949f2a2a3cef3da7a909e 2024-11-12T18:15:51,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/d428f528ae1148e3895737b76e103629 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/d428f528ae1148e3895737b76e103629 2024-11-12T18:15:51,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/774493d0b1ca47c79506a4112831b0cb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/774493d0b1ca47c79506a4112831b0cb 2024-11-12T18:15:51,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e25bc85a82914726b4764d40fe4d7549, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6070f99865a7459fbf92ad9f9d15a61c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/954c39a213a446a48f6cfa298ff6819e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6b10e3eb16314d638018e5d0bd7c93dd, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/732366763ebe4ac184d5102ba259431c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/9266fd5101464273834add093eee5806, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/be9aa15cd29f447d86de2a83160abe84, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/2742bcf231e048758557d3d0a7e5e16d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/0192a76138c7440a9e5605db2dd7b1b9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/af06de348b854350bf36870014143360, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a6617d611c5f4f9ab8077d68d3226b0f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/76df9ab48693416aa317e629bea88b57, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/985987de29c24f8b9a84ed2e9e4aeeaf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/cb38ac6cdeb3432e92229c947c28006f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/1754b0aad0154ef4a366d37a2e79cd8e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/4869c7d89b8b4f639e36de8bdd7f7f27, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a0ef1f3830d845b28fb39c24a812e542, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/95007d915faf4f6285819f5b5ea67c39, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e5061360aed0407b9dcab9a8f03439ea, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/84e4925bfd8e4703bc3b5297069f50ce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6457cb893d664be4adfdb3bc82c967c3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/81ecadf7313244c7ba4a42e5f57e665f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/5cf201df8fa64a1aa1fc639f3418ba20] to archive 2024-11-12T18:15:51,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:15:51,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e25bc85a82914726b4764d40fe4d7549 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e25bc85a82914726b4764d40fe4d7549 2024-11-12T18:15:51,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6070f99865a7459fbf92ad9f9d15a61c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6070f99865a7459fbf92ad9f9d15a61c 2024-11-12T18:15:51,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/954c39a213a446a48f6cfa298ff6819e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/954c39a213a446a48f6cfa298ff6819e 2024-11-12T18:15:51,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6b10e3eb16314d638018e5d0bd7c93dd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6b10e3eb16314d638018e5d0bd7c93dd 2024-11-12T18:15:51,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/732366763ebe4ac184d5102ba259431c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/732366763ebe4ac184d5102ba259431c 2024-11-12T18:15:51,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/9266fd5101464273834add093eee5806 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/9266fd5101464273834add093eee5806 2024-11-12T18:15:51,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/be9aa15cd29f447d86de2a83160abe84 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/be9aa15cd29f447d86de2a83160abe84 2024-11-12T18:15:51,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/2742bcf231e048758557d3d0a7e5e16d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/2742bcf231e048758557d3d0a7e5e16d 2024-11-12T18:15:51,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/0192a76138c7440a9e5605db2dd7b1b9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/0192a76138c7440a9e5605db2dd7b1b9 2024-11-12T18:15:51,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/af06de348b854350bf36870014143360 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/af06de348b854350bf36870014143360 2024-11-12T18:15:51,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a6617d611c5f4f9ab8077d68d3226b0f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a6617d611c5f4f9ab8077d68d3226b0f 2024-11-12T18:15:51,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/76df9ab48693416aa317e629bea88b57 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/76df9ab48693416aa317e629bea88b57 2024-11-12T18:15:51,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/985987de29c24f8b9a84ed2e9e4aeeaf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/985987de29c24f8b9a84ed2e9e4aeeaf 2024-11-12T18:15:51,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/cb38ac6cdeb3432e92229c947c28006f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/cb38ac6cdeb3432e92229c947c28006f 2024-11-12T18:15:51,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/1754b0aad0154ef4a366d37a2e79cd8e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/1754b0aad0154ef4a366d37a2e79cd8e 2024-11-12T18:15:51,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/4869c7d89b8b4f639e36de8bdd7f7f27 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/4869c7d89b8b4f639e36de8bdd7f7f27 2024-11-12T18:15:51,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a0ef1f3830d845b28fb39c24a812e542 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/a0ef1f3830d845b28fb39c24a812e542 2024-11-12T18:15:51,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/95007d915faf4f6285819f5b5ea67c39 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/95007d915faf4f6285819f5b5ea67c39 2024-11-12T18:15:51,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e5061360aed0407b9dcab9a8f03439ea to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/e5061360aed0407b9dcab9a8f03439ea 2024-11-12T18:15:51,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/84e4925bfd8e4703bc3b5297069f50ce to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/84e4925bfd8e4703bc3b5297069f50ce 2024-11-12T18:15:51,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6457cb893d664be4adfdb3bc82c967c3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/6457cb893d664be4adfdb3bc82c967c3 2024-11-12T18:15:51,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/81ecadf7313244c7ba4a42e5f57e665f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/81ecadf7313244c7ba4a42e5f57e665f 2024-11-12T18:15:51,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/5cf201df8fa64a1aa1fc639f3418ba20 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/5cf201df8fa64a1aa1fc639f3418ba20 2024-11-12T18:15:51,112 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/recovered.edits/394.seqid, newMaxSeqId=394, maxSeqId=4 2024-11-12T18:15:51,113 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. 2024-11-12T18:15:51,113 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 3946528656bca5567e369f3af446cf65: Waiting for close lock at 1731435349377Running coprocessor pre-close hooks at 1731435349377Disabling compacts and flushes for region at 1731435349377Disabling writes for close at 1731435349377Obtaining lock to block concurrent updates at 1731435349377Preparing flush snapshotting stores in 3946528656bca5567e369f3af446cf65 at 1731435349377Finished memstore snapshotting TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731435349378 (+1 ms)Flushing stores of TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65. at 1731435349378Flushing 3946528656bca5567e369f3af446cf65/A: creating writer at 1731435349378Flushing mob file 3946528656bca5567e369f3af446cf65/A: appending metadata at 1731435349383 (+5 ms)Flushing mob file 3946528656bca5567e369f3af446cf65/A: closing flushed file at 1731435349383Flushing 3946528656bca5567e369f3af446cf65/A: appending metadata at 1731435349791 (+408 ms)Flushing 3946528656bca5567e369f3af446cf65/A: closing flushed file at 1731435349791Flushing 3946528656bca5567e369f3af446cf65/B: creating writer at 1731435350199 (+408 ms)Flushing 3946528656bca5567e369f3af446cf65/B: appending metadata at 1731435350202 (+3 ms)Flushing 3946528656bca5567e369f3af446cf65/B: closing flushed file at 1731435350202Flushing 3946528656bca5567e369f3af446cf65/C: creating writer at 1731435350611 (+409 ms)Flushing 3946528656bca5567e369f3af446cf65/C: appending metadata at 1731435350613 (+2 ms)Flushing 3946528656bca5567e369f3af446cf65/C: closing flushed file at 1731435350614 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fd824ed: reopening flushed file at 1731435351022 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66a6fcad: reopening flushed file at 1731435351027 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d101b99: reopening flushed file at 1731435351031 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3946528656bca5567e369f3af446cf65 in 1659ms, sequenceid=391, compaction requested=true at 1731435351036 (+5 ms)Writing region close event to WAL at 1731435351110 (+74 ms)Running coprocessor post-close hooks at 1731435351113 (+3 ms)Closed at 1731435351113 2024-11-12T18:15:51,115 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3946528656bca5567e369f3af446cf65, regionState=CLOSED 2024-11-12T18:15:51,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:15:51,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-12T18:15:51,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 3946528656bca5567e369f3af446cf65, server=9911683f163c,43757,1731435233300 in 1.8930 sec 2024-11-12T18:15:51,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-12T18:15:51,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3946528656bca5567e369f3af446cf65, UNASSIGN in 1.8980 sec 2024-11-12T18:15:51,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T18:15:51,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-12T18:15:51,126 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435351125"}]},"ts":"1731435351125"} 2024-11-12T18:15:51,127 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T18:15:51,127 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T18:15:51,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9130 sec 2024-11-12T18:15:51,242 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T18:15:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T18:15:51,361 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:51,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T18:15:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,363 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T18:15:51,364 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,367 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/recovered.edits] 2024-11-12T18:15:51,373 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c68e21ac7124922b5fc28372f366c47 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/7c68e21ac7124922b5fc28372f366c47 2024-11-12T18:15:51,374 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/b5a497e92e37404cb1f8ab9d65024d8e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/b5a497e92e37404cb1f8ab9d65024d8e 2024-11-12T18:15:51,375 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c186da54b82347d69f52e478c28013c9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/A/c186da54b82347d69f52e478c28013c9 2024-11-12T18:15:51,377 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0d47b6d0be2f4e54ad2f9fed4a831fb7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/0d47b6d0be2f4e54ad2f9fed4a831fb7 2024-11-12T18:15:51,378 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8d55b5c6619349beb93034e526684c7e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/8d55b5c6619349beb93034e526684c7e 2024-11-12T18:15:51,380 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/fa0eea92220643519049deda8b0bab89 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/B/fa0eea92220643519049deda8b0bab89 2024-11-12T18:15:51,382 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/52243141ddfe4360836e73d2d22b4111 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/52243141ddfe4360836e73d2d22b4111 2024-11-12T18:15:51,383 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/66cd72b3c27747cdaacea4d4204bdfdb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/66cd72b3c27747cdaacea4d4204bdfdb 2024-11-12T18:15:51,384 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/c36a82bd60cb407a9dc6059a4fb77927 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/C/c36a82bd60cb407a9dc6059a4fb77927 2024-11-12T18:15:51,387 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/recovered.edits/394.seqid to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65/recovered.edits/394.seqid 2024-11-12T18:15:51,387 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,387 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T18:15:51,388 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T18:15:51,388 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T18:15:51,391 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120551443e63404344b834316d48d69934_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120551443e63404344b834316d48d69934_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,392 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111219db00ca47ed470f8ffd869ae3139f45_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111219db00ca47ed470f8ffd869ae3139f45_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,395 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121cd2ecb1c8cf4490a285a9502c2526f5_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121cd2ecb1c8cf4490a285a9502c2526f5_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,396 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121f4e2ea1778e47998597f6513f571096_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121f4e2ea1778e47998597f6513f571096_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,398 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122446393345de443f83ede725e6824275_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122446393345de443f83ede725e6824275_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,399 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112365c8bcc41494b069d9f78ffc7369432_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112365c8bcc41494b069d9f78ffc7369432_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,401 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111243f600d3d6ea45e58ecaad52fd3ead14_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111243f600d3d6ea45e58ecaad52fd3ead14_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,402 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124c1caa8737aa49d9b5f6bf0cf2da21eb_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124c1caa8737aa49d9b5f6bf0cf2da21eb_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,403 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111253b381198d9548f4ae87dc49d66b1a75_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111253b381198d9548f4ae87dc49d66b1a75_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,403 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111256bfeafb1302411b8d758e886d91c935_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111256bfeafb1302411b8d758e886d91c935_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,404 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111268d4484ac2a348c2bff0bb4e4b0d0ff9_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111268d4484ac2a348c2bff0bb4e4b0d0ff9_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,406 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d9b8a8fe49f42aa985ab3888530350c_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d9b8a8fe49f42aa985ab3888530350c_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,407 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127453196a039b4414bec133b1cd276f4e_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127453196a039b4414bec133b1cd276f4e_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,408 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112904632d4e2e845629c2bbcd9336ef2e5_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112904632d4e2e845629c2bbcd9336ef2e5_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,409 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129824b4a4f396436296a81dbfd006a096_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129824b4a4f396436296a81dbfd006a096_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,410 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bb7e7ef113d64ae9a79dfb3cc3e98f53_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112bb7e7ef113d64ae9a79dfb3cc3e98f53_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,411 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d09369a185c24523b13188a92439cf52_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d09369a185c24523b13188a92439cf52_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,413 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eb466ed1f6f8465d88c15853cc32d01f_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eb466ed1f6f8465d88c15853cc32d01f_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,413 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112faeed73101c84493857701a19bb084fa_3946528656bca5567e369f3af446cf65 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112faeed73101c84493857701a19bb084fa_3946528656bca5567e369f3af446cf65 2024-11-12T18:15:51,414 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T18:15:51,416 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,418 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T18:15:51,420 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T18:15:51,422 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,422 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T18:15:51,422 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731435351422"}]},"ts":"9223372036854775807"} 2024-11-12T18:15:51,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T18:15:51,424 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3946528656bca5567e369f3af446cf65, NAME => 'TestAcidGuarantees,,1731435323925.3946528656bca5567e369f3af446cf65.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T18:15:51,424 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T18:15:51,424 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731435351424"}]},"ts":"9223372036854775807"} 2024-11-12T18:15:51,426 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T18:15:51,427 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-12T18:15:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T18:15:51,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T18:15:51,471 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:51,482 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=459 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=755 (was 654) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1643 (was 2128) 2024-11-12T18:15:51,492 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=755, ProcessCount=11, AvailableMemoryMB=1642 2024-11-12T18:15:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:15:51,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:15:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T18:15:51,496 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:15:51,496 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:51,496 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-12T18:15:51,497 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:15:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T18:15:51,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742289_1465 (size=1061) 2024-11-12T18:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T18:15:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T18:15:51,904 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:15:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742290_1466 (size=53) 2024-11-12T18:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T18:15:52,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:15:52,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0f73c86ec05d2a0b97b6becb163767f7, disabling compactions & flushes 2024-11-12T18:15:52,311 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. after waiting 0 ms 2024-11-12T18:15:52,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,311 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,311 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f73c86ec05d2a0b97b6becb163767f7: Waiting for close lock at 1731435352311Disabling compacts and flushes for region at 1731435352311Disabling writes for close at 1731435352311Writing region close event to WAL at 1731435352311Closed at 1731435352311 2024-11-12T18:15:52,312 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:15:52,313 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731435352313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435352313"}]},"ts":"1731435352313"} 2024-11-12T18:15:52,314 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T18:15:52,315 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:15:52,315 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435352315"}]},"ts":"1731435352315"} 2024-11-12T18:15:52,316 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T18:15:52,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, ASSIGN}] 2024-11-12T18:15:52,317 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, ASSIGN 2024-11-12T18:15:52,318 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, ASSIGN; state=OFFLINE, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=false 2024-11-12T18:15:52,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=0f73c86ec05d2a0b97b6becb163767f7, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:52,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, ASSIGN because future has completed 2024-11-12T18:15:52,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:15:52,625 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,626 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:15:52,627 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,627 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:15:52,627 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,627 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,628 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,629 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:52,629 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f73c86ec05d2a0b97b6becb163767f7 columnFamilyName A 2024-11-12T18:15:52,630 DEBUG [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:52,630 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.HStore(327): Store=0f73c86ec05d2a0b97b6becb163767f7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:52,630 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T18:15:52,631 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:52,631 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f73c86ec05d2a0b97b6becb163767f7 columnFamilyName B 2024-11-12T18:15:52,631 DEBUG [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:52,632 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.HStore(327): Store=0f73c86ec05d2a0b97b6becb163767f7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:52,632 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,633 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:15:52,633 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f73c86ec05d2a0b97b6becb163767f7 columnFamilyName C 2024-11-12T18:15:52,633 DEBUG [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:15:52,633 INFO [StoreOpener-0f73c86ec05d2a0b97b6becb163767f7-1 {}] regionserver.HStore(327): Store=0f73c86ec05d2a0b97b6becb163767f7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:15:52,634 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,634 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,634 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,634 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,635 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,635 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,636 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:15:52,637 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,639 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:15:52,639 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 0f73c86ec05d2a0b97b6becb163767f7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68327572, jitterRate=0.01816016435623169}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:15:52,639 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:52,640 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 0f73c86ec05d2a0b97b6becb163767f7: Running coprocessor pre-open hook at 1731435352627Writing region info on filesystem at 1731435352627Initializing all the Stores at 1731435352627Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435352628 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435352628Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435352628Cleaning up temporary data from old regions at 1731435352635 (+7 ms)Running coprocessor post-open hooks at 1731435352639 (+4 ms)Region opened successfully at 1731435352640 (+1 ms) 2024-11-12T18:15:52,641 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., pid=118, masterSystemTime=1731435352622 2024-11-12T18:15:52,643 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,643 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:52,644 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=0f73c86ec05d2a0b97b6becb163767f7, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:15:52,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:15:52,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-12T18:15:52,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 in 175 msec 2024-11-12T18:15:52,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-12T18:15:52,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, ASSIGN in 332 msec 2024-11-12T18:15:52,651 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:15:52,651 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435352651"}]},"ts":"1731435352651"} 2024-11-12T18:15:52,652 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T18:15:52,653 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:15:52,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-12T18:15:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-12T18:15:53,640 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:53,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,643 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1114fedb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,644 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24fe461a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,645 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,648 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:15:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:15:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d002574 2024-11-12T18:15:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:15:53,649 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:15:53,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccc76de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,653 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ea39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,655 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15091808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb91324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,659 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe13a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,660 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6083a78a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be1e24d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658e4b3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@790540dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcdf405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e880099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f244587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc06cd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407d2e9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5969a2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,675 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d536e1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,677 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794d2dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,678 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a29a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:15:53,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:15:53,680 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52bf1cb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:15:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:15:53,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:15:53,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de5e22e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:15:53,685 DEBUG [Thread-2084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,685 DEBUG [Thread-2099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:53,685 DEBUG [Thread-2088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,686 DEBUG [Thread-2086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,686 DEBUG [Thread-2095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,686 DEBUG [Thread-2101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-12T18:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T18:15:53,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,690 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:53,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,692 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:53,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,697 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,697 DEBUG [Thread-2090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,698 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,700 DEBUG [Thread-2092 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,700 DEBUG [Thread-2092 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:53,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:15:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:53,712 DEBUG [Thread-2097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:15:53,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:15:53,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:15:53,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:15:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2] 2024-11-12T18:15:53,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/8a770d01d91f425195a330293bf3f453 is 50, key is test_row_0/A:col10/1731435353707/Put/seqid=0 2024-11-12T18:15:53,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742291_1467 (size=12001) 2024-11-12T18:15:53,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435353748, max timestamp=1731435353766Num uniques 859; 2024-11-12T18:15:53,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435353748, max timestamp=1731435353766Num uniques 859; 2024-11-12T18:15:53,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:53,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:53,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/8a770d01d91f425195a330293bf3f453 2024-11-12T18:15:53,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435353748, max timestamp=1731435353766Num uniques 859; 2024-11-12T18:15:53,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:53,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/9db373f9590c47f3b934e2b466ec5ea9 is 50, key is test_row_0/B:col10/1731435353707/Put/seqid=0 2024-11-12T18:15:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T18:15:53,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742292_1468 (size=12001) 2024-11-12T18:15:53,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/9db373f9590c47f3b934e2b466ec5ea9 2024-11-12T18:15:53,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435413803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435413804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435413806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435413808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435413810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/6afeb3b33d9d482a836e517d6a83ad1e is 50, key is test_row_0/C:col10/1731435353707/Put/seqid=0 2024-11-12T18:15:53,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742293_1469 (size=12001) 2024-11-12T18:15:53,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/6afeb3b33d9d482a836e517d6a83ad1e 2024-11-12T18:15:53,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T18:15:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:53,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/8a770d01d91f425195a330293bf3f453 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8a770d01d91f425195a330293bf3f453 2024-11-12T18:15:53,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8a770d01d91f425195a330293bf3f453, entries=150, sequenceid=11, filesize=11.7 K 2024-11-12T18:15:53,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/9db373f9590c47f3b934e2b466ec5ea9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/9db373f9590c47f3b934e2b466ec5ea9 2024-11-12T18:15:53,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/9db373f9590c47f3b934e2b466ec5ea9, entries=150, sequenceid=11, filesize=11.7 K 2024-11-12T18:15:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/6afeb3b33d9d482a836e517d6a83ad1e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6afeb3b33d9d482a836e517d6a83ad1e 2024-11-12T18:15:53,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6afeb3b33d9d482a836e517d6a83ad1e, entries=150, sequenceid=11, filesize=11.7 K 2024-11-12T18:15:53,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0f73c86ec05d2a0b97b6becb163767f7 in 153ms, sequenceid=11, compaction requested=false 2024-11-12T18:15:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:53,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-12T18:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:53,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/efcb73745241444d9fa10fbd0c988eb2 is 50, key is test_row_0/A:col10/1731435353927/Put/seqid=0 2024-11-12T18:15:53,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742294_1470 (size=14341) 2024-11-12T18:15:53,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/efcb73745241444d9fa10fbd0c988eb2 2024-11-12T18:15:53,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4a3f2eaf194d496ab58132ce252a4597 is 50, key is test_row_0/B:col10/1731435353927/Put/seqid=0 2024-11-12T18:15:53,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742295_1471 (size=12001) 2024-11-12T18:15:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435413945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435413946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435413948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435413948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435413959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:53,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T18:15:54,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:54,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T18:15:54,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435414070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435414070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435414070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435414070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435414081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T18:15:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435414293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435414293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435414293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T18:15:54,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:54,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435414297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:54,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435414304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T18:15:54,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4a3f2eaf194d496ab58132ce252a4597 2024-11-12T18:15:54,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/744b0b87fe8e48c29c519370361bd9dc is 50, key is test_row_0/C:col10/1731435353927/Put/seqid=0 2024-11-12T18:15:54,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742296_1472 (size=12001) 2024-11-12T18:15:54,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/744b0b87fe8e48c29c519370361bd9dc 2024-11-12T18:15:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/efcb73745241444d9fa10fbd0c988eb2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/efcb73745241444d9fa10fbd0c988eb2 2024-11-12T18:15:54,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/efcb73745241444d9fa10fbd0c988eb2, entries=200, sequenceid=48, filesize=14.0 K 2024-11-12T18:15:54,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4a3f2eaf194d496ab58132ce252a4597 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a3f2eaf194d496ab58132ce252a4597 2024-11-12T18:15:54,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a3f2eaf194d496ab58132ce252a4597, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T18:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/744b0b87fe8e48c29c519370361bd9dc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/744b0b87fe8e48c29c519370361bd9dc 2024-11-12T18:15:54,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/744b0b87fe8e48c29c519370361bd9dc, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T18:15:54,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 0f73c86ec05d2a0b97b6becb163767f7 in 482ms, sequenceid=48, compaction requested=false 2024-11-12T18:15:54,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:54,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:54,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/da2037b2350c4298915734346d6f8f74 is 50, key is test_row_0/A:col10/1731435353941/Put/seqid=0 2024-11-12T18:15:54,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742297_1473 (size=12001) 2024-11-12T18:15:54,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/da2037b2350c4298915734346d6f8f74 2024-11-12T18:15:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/de96397acb8b4ad799f32fe7ce15a35e is 50, key is test_row_0/B:col10/1731435353941/Put/seqid=0 2024-11-12T18:15:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742298_1474 (size=12001) 2024-11-12T18:15:54,495 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/de96397acb8b4ad799f32fe7ce15a35e 2024-11-12T18:15:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/81dafe16dc334217a3aea37f11796eda is 50, key is test_row_0/C:col10/1731435353941/Put/seqid=0 2024-11-12T18:15:54,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742299_1475 (size=12001) 2024-11-12T18:15:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:54,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:54,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731435354628, max timestamp=1731435354662Num uniques 859; 2024-11-12T18:15:54,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T18:15:54,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731435354628, max timestamp=1731435354662Num uniques 859; 2024-11-12T18:15:54,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T18:15:54,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731435354628, max timestamp=1731435354662Num uniques 859; 2024-11-12T18:15:54,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T18:15:54,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435414700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435414701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435414702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435414704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435414716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435414820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T18:15:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435414832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435414832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435414832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435414834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:54,960 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/81dafe16dc334217a3aea37f11796eda 2024-11-12T18:15:54,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/da2037b2350c4298915734346d6f8f74 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/da2037b2350c4298915734346d6f8f74 2024-11-12T18:15:54,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/da2037b2350c4298915734346d6f8f74, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T18:15:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/de96397acb8b4ad799f32fe7ce15a35e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/de96397acb8b4ad799f32fe7ce15a35e 2024-11-12T18:15:54,981 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/de96397acb8b4ad799f32fe7ce15a35e, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T18:15:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/81dafe16dc334217a3aea37f11796eda as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/81dafe16dc334217a3aea37f11796eda 2024-11-12T18:15:54,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/81dafe16dc334217a3aea37f11796eda, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T18:15:54,987 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 0f73c86ec05d2a0b97b6becb163767f7 in 525ms, sequenceid=57, compaction requested=true 2024-11-12T18:15:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-12T18:15:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-12T18:15:54,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T18:15:54,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2960 sec 2024-11-12T18:15:54,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.3060 sec 2024-11-12T18:15:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:55,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-12T18:15:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:55,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/7b5453110e7c48fca4912164c780e7c6 is 50, key is test_row_0/A:col10/1731435354714/Put/seqid=0 2024-11-12T18:15:55,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742300_1476 (size=14341) 2024-11-12T18:15:55,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/7b5453110e7c48fca4912164c780e7c6 2024-11-12T18:15:55,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4a5ce3b44f1c45e4a909d463fd760818 is 50, key is test_row_0/B:col10/1731435354714/Put/seqid=0 2024-11-12T18:15:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742301_1477 (size=12001) 2024-11-12T18:15:55,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4a5ce3b44f1c45e4a909d463fd760818 2024-11-12T18:15:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435415059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435415060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435415060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435415065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/a5cd9e06cbb04202a8b7068e36dc0f2f is 50, key is test_row_0/C:col10/1731435354714/Put/seqid=0 2024-11-12T18:15:55,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742302_1478 (size=12001) 2024-11-12T18:15:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435415076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435415180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435415180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435415200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435415381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435415391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435415391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435415400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435415420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/a5cd9e06cbb04202a8b7068e36dc0f2f 2024-11-12T18:15:55,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/7b5453110e7c48fca4912164c780e7c6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/7b5453110e7c48fca4912164c780e7c6 2024-11-12T18:15:55,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/7b5453110e7c48fca4912164c780e7c6, entries=200, sequenceid=94, filesize=14.0 K 2024-11-12T18:15:55,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4a5ce3b44f1c45e4a909d463fd760818 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a5ce3b44f1c45e4a909d463fd760818 2024-11-12T18:15:55,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a5ce3b44f1c45e4a909d463fd760818, entries=150, sequenceid=94, filesize=11.7 K 2024-11-12T18:15:55,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/a5cd9e06cbb04202a8b7068e36dc0f2f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a5cd9e06cbb04202a8b7068e36dc0f2f 2024-11-12T18:15:55,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a5cd9e06cbb04202a8b7068e36dc0f2f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-12T18:15:55,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 0f73c86ec05d2a0b97b6becb163767f7 in 454ms, sequenceid=94, compaction requested=true 2024-11-12T18:15:55,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:55,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:55,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:55,505 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:55,505 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:55,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:55,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:55,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:55,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:55,506 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:55,506 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:15:55,506 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:55,507 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/9db373f9590c47f3b934e2b466ec5ea9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a3f2eaf194d496ab58132ce252a4597, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/de96397acb8b4ad799f32fe7ce15a35e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a5ce3b44f1c45e4a909d463fd760818] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=46.9 K 2024-11-12T18:15:55,507 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:55,507 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:15:55,507 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:55,507 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8a770d01d91f425195a330293bf3f453, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/efcb73745241444d9fa10fbd0c988eb2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/da2037b2350c4298915734346d6f8f74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/7b5453110e7c48fca4912164c780e7c6] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=51.4 K 2024-11-12T18:15:55,507 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db373f9590c47f3b934e2b466ec5ea9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731435353697 2024-11-12T18:15:55,508 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a770d01d91f425195a330293bf3f453, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731435353697 2024-11-12T18:15:55,508 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3f2eaf194d496ab58132ce252a4597, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731435353788 2024-11-12T18:15:55,508 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting efcb73745241444d9fa10fbd0c988eb2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731435353788 2024-11-12T18:15:55,508 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting de96397acb8b4ad799f32fe7ce15a35e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731435353941 2024-11-12T18:15:55,509 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2037b2350c4298915734346d6f8f74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731435353941 2024-11-12T18:15:55,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5ce3b44f1c45e4a909d463fd760818, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731435354702 2024-11-12T18:15:55,509 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5453110e7c48fca4912164c780e7c6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731435354700 2024-11-12T18:15:55,520 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:55,521 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ad419212dca04383a94ca5f362b5d621 is 50, key is test_row_0/B:col10/1731435354714/Put/seqid=0 2024-11-12T18:15:55,523 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:55,524 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/21d74a3e2785407f9a4c599a0f9b5a12 is 50, key is test_row_0/A:col10/1731435354714/Put/seqid=0 2024-11-12T18:15:55,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742304_1480 (size=12139) 2024-11-12T18:15:55,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742303_1479 (size=12139) 2024-11-12T18:15:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:55,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:15:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:55,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/ca22f23ce52946e9b3a1adab19e6b113 is 50, key is test_row_0/A:col10/1731435355074/Put/seqid=0 2024-11-12T18:15:55,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742305_1481 (size=12001) 2024-11-12T18:15:55,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435355795, max timestamp=1731435355824Num uniques 859; 2024-11-12T18:15:55,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435355795, max timestamp=1731435355824Num uniques 859; 2024-11-12T18:15:55,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:55,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:55,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435355795, max timestamp=1731435355824Num uniques 859; 2024-11-12T18:15:55,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:15:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T18:15:55,851 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:15:55,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:15:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T18:15:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:15:55,855 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:15:55,856 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:15:55,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:15:55,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435415904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435415905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435415910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435415911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435415913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:55,938 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/21d74a3e2785407f9a4c599a0f9b5a12 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/21d74a3e2785407f9a4c599a0f9b5a12 2024-11-12T18:15:55,943 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ad419212dca04383a94ca5f362b5d621 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ad419212dca04383a94ca5f362b5d621 2024-11-12T18:15:55,943 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into 21d74a3e2785407f9a4c599a0f9b5a12(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:55,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:55,943 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=12, startTime=1731435355505; duration=0sec 2024-11-12T18:15:55,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:55,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:15:55,943 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:55,945 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:55,945 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:15:55,945 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:55,945 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6afeb3b33d9d482a836e517d6a83ad1e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/744b0b87fe8e48c29c519370361bd9dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/81dafe16dc334217a3aea37f11796eda, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a5cd9e06cbb04202a8b7068e36dc0f2f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=46.9 K 2024-11-12T18:15:55,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6afeb3b33d9d482a836e517d6a83ad1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731435353697 2024-11-12T18:15:55,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744b0b87fe8e48c29c519370361bd9dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731435353788 2024-11-12T18:15:55,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81dafe16dc334217a3aea37f11796eda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731435353941 2024-11-12T18:15:55,946 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5cd9e06cbb04202a8b7068e36dc0f2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731435354702 2024-11-12T18:15:55,949 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into ad419212dca04383a94ca5f362b5d621(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:55,949 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:55,949 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=12, startTime=1731435355505; duration=0sec 2024-11-12T18:15:55,949 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:55,949 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:15:55,958 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#404 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:55,958 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/562dc07d1fea4c35bd2a4acfe92f20ba is 50, key is test_row_0/C:col10/1731435354714/Put/seqid=0 2024-11-12T18:15:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:15:55,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742306_1482 (size=12139) 2024-11-12T18:15:55,971 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/562dc07d1fea4c35bd2a4acfe92f20ba as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/562dc07d1fea4c35bd2a4acfe92f20ba 2024-11-12T18:15:55,979 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into 562dc07d1fea4c35bd2a4acfe92f20ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:55,979 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:55,979 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=12, startTime=1731435355505; duration=0sec 2024-11-12T18:15:55,979 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:55,979 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:15:56,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T18:15:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435416030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435416031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435416040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435416040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/ca22f23ce52946e9b3a1adab19e6b113 2024-11-12T18:15:56,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/fa2114f3438d4797a3f30d05a565d41b is 50, key is test_row_0/B:col10/1731435355074/Put/seqid=0 2024-11-12T18:15:56,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742307_1483 (size=12001) 2024-11-12T18:15:56,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T18:15:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:15:56,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435416250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435416251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435416251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435416260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T18:15:56,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:56,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T18:15:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:15:56,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/fa2114f3438d4797a3f30d05a565d41b 2024-11-12T18:15:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435416561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435416561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435416561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/43a36d0263004d0ebca66514fb2c9f46 is 50, key is test_row_0/C:col10/1731435355074/Put/seqid=0 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435416580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742308_1484 (size=12001) 2024-11-12T18:15:56,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/43a36d0263004d0ebca66514fb2c9f46 2024-11-12T18:15:56,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/ca22f23ce52946e9b3a1adab19e6b113 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/ca22f23ce52946e9b3a1adab19e6b113 2024-11-12T18:15:56,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/ca22f23ce52946e9b3a1adab19e6b113, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T18:15:56,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/fa2114f3438d4797a3f30d05a565d41b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/fa2114f3438d4797a3f30d05a565d41b 2024-11-12T18:15:56,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/fa2114f3438d4797a3f30d05a565d41b, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T18:15:56,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/43a36d0263004d0ebca66514fb2c9f46 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/43a36d0263004d0ebca66514fb2c9f46 2024-11-12T18:15:56,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T18:15:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:15:56,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/43a36d0263004d0ebca66514fb2c9f46, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T18:15:56,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0f73c86ec05d2a0b97b6becb163767f7 in 912ms, sequenceid=104, compaction requested=false 2024-11-12T18:15:56,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:56,641 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:15:56,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T18:15:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:56,785 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T18:15:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/aac4ce1882084cf6895097ded2085fc8 is 50, key is test_row_0/A:col10/1731435355901/Put/seqid=0 2024-11-12T18:15:56,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742309_1485 (size=12151) 2024-11-12T18:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:56,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:15:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:15:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435417042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435417083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435417084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435417085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435417101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435417170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,210 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/aac4ce1882084cf6895097ded2085fc8 2024-11-12T18:15:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/e710e8887b0445dda98de5bbdc13d786 is 50, key is test_row_0/B:col10/1731435355901/Put/seqid=0 2024-11-12T18:15:57,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742310_1486 (size=12151) 2024-11-12T18:15:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435417391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:57,633 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/e710e8887b0445dda98de5bbdc13d786 2024-11-12T18:15:57,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/03505cf7fde640908f34e736feadee87 is 50, key is test_row_0/C:col10/1731435355901/Put/seqid=0 2024-11-12T18:15:57,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742311_1487 (size=12151) 2024-11-12T18:15:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435417700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:15:58,063 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/03505cf7fde640908f34e736feadee87 2024-11-12T18:15:58,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/aac4ce1882084cf6895097ded2085fc8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/aac4ce1882084cf6895097ded2085fc8 2024-11-12T18:15:58,075 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/aac4ce1882084cf6895097ded2085fc8, entries=150, sequenceid=143, filesize=11.9 K 2024-11-12T18:15:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/e710e8887b0445dda98de5bbdc13d786 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/e710e8887b0445dda98de5bbdc13d786 2024-11-12T18:15:58,081 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/e710e8887b0445dda98de5bbdc13d786, entries=150, sequenceid=143, filesize=11.9 K 2024-11-12T18:15:58,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/03505cf7fde640908f34e736feadee87 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/03505cf7fde640908f34e736feadee87 2024-11-12T18:15:58,087 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/03505cf7fde640908f34e736feadee87, entries=150, sequenceid=143, filesize=11.9 K 2024-11-12T18:15:58,091 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 0f73c86ec05d2a0b97b6becb163767f7 in 1306ms, sequenceid=143, compaction requested=true 2024-11-12T18:15:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T18:15:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T18:15:58,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T18:15:58,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2380 sec 2024-11-12T18:15:58,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2430 sec 2024-11-12T18:15:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:58,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:15:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:15:58,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/0e16597dfd5648c3bc6c0849c4907a6f is 50, key is test_row_0/A:col10/1731435358100/Put/seqid=0 2024-11-12T18:15:58,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742312_1488 (size=14541) 2024-11-12T18:15:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435358181, max timestamp=1731435358199Num uniques 859; 2024-11-12T18:15:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731435358181, max timestamp=1731435358199Num uniques 859; 2024-11-12T18:15:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:58,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:58,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731435358181, max timestamp=1731435358199Num uniques 859; 2024-11-12T18:15:58,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:15:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435418239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435418240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435418241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435418244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435418256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435418360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435418361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435418361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435418361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435418373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/0e16597dfd5648c3bc6c0849c4907a6f 2024-11-12T18:15:58,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/b9bf79483420417ba89aa198e690865b is 50, key is test_row_0/B:col10/1731435358100/Put/seqid=0 2024-11-12T18:15:58,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742313_1489 (size=12151) 2024-11-12T18:15:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435418581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435418581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435418581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435418584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435418590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435418900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435418900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435418900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435418900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435418910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:58,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/b9bf79483420417ba89aa198e690865b 2024-11-12T18:15:58,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/e586ff8e178d4b409439ae8da7fd0eae is 50, key is test_row_0/C:col10/1731435358100/Put/seqid=0 2024-11-12T18:15:58,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742314_1490 (size=12151) 2024-11-12T18:15:58,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/e586ff8e178d4b409439ae8da7fd0eae 2024-11-12T18:15:58,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/0e16597dfd5648c3bc6c0849c4907a6f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/0e16597dfd5648c3bc6c0849c4907a6f 2024-11-12T18:15:58,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/0e16597dfd5648c3bc6c0849c4907a6f, entries=200, sequenceid=154, filesize=14.2 K 2024-11-12T18:15:58,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/b9bf79483420417ba89aa198e690865b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/b9bf79483420417ba89aa198e690865b 2024-11-12T18:15:58,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/b9bf79483420417ba89aa198e690865b, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T18:15:58,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/e586ff8e178d4b409439ae8da7fd0eae as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e586ff8e178d4b409439ae8da7fd0eae 2024-11-12T18:15:58,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e586ff8e178d4b409439ae8da7fd0eae, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T18:15:58,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0f73c86ec05d2a0b97b6becb163767f7 in 886ms, sequenceid=154, compaction requested=true 2024-11-12T18:15:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:15:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:58,989 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:15:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:15:58,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:15:58,990 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:58,993 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50832 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:58,993 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:58,993 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:15:58,993 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:15:58,993 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:58,993 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:58,993 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/21d74a3e2785407f9a4c599a0f9b5a12, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/ca22f23ce52946e9b3a1adab19e6b113, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/aac4ce1882084cf6895097ded2085fc8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/0e16597dfd5648c3bc6c0849c4907a6f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=49.6 K 2024-11-12T18:15:58,993 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ad419212dca04383a94ca5f362b5d621, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/fa2114f3438d4797a3f30d05a565d41b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/e710e8887b0445dda98de5bbdc13d786, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/b9bf79483420417ba89aa198e690865b] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=47.3 K 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ad419212dca04383a94ca5f362b5d621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731435354702 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d74a3e2785407f9a4c599a0f9b5a12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731435354702 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2114f3438d4797a3f30d05a565d41b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731435355074 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca22f23ce52946e9b3a1adab19e6b113, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731435355074 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac4ce1882084cf6895097ded2085fc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731435355876 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e710e8887b0445dda98de5bbdc13d786, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731435355876 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bf79483420417ba89aa198e690865b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731435356984 2024-11-12T18:15:58,994 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e16597dfd5648c3bc6c0849c4907a6f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731435356984 2024-11-12T18:15:59,005 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:59,005 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/7db937547ad840f9af7fd2ab6087aeb0 is 50, key is test_row_0/B:col10/1731435358100/Put/seqid=0 2024-11-12T18:15:59,009 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:59,010 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/abdda80d3fac48e89909801d693b788b is 50, key is test_row_0/A:col10/1731435358100/Put/seqid=0 2024-11-12T18:15:59,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742315_1491 (size=12425) 2024-11-12T18:15:59,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742316_1492 (size=12425) 2024-11-12T18:15:59,423 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/7db937547ad840f9af7fd2ab6087aeb0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/7db937547ad840f9af7fd2ab6087aeb0 2024-11-12T18:15:59,429 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/abdda80d3fac48e89909801d693b788b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/abdda80d3fac48e89909801d693b788b 2024-11-12T18:15:59,432 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into 7db937547ad840f9af7fd2ab6087aeb0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:59,432 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:59,432 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=12, startTime=1731435358988; duration=0sec 2024-11-12T18:15:59,432 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:15:59,432 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:15:59,432 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:15:59,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-12T18:15:59,434 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into abdda80d3fac48e89909801d693b788b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:59,434 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:59,434 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=12, startTime=1731435358988; duration=0sec 2024-11-12T18:15:59,434 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:59,434 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:15:59,434 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:15:59,434 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:15:59,435 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:15:59,435 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/562dc07d1fea4c35bd2a4acfe92f20ba, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/43a36d0263004d0ebca66514fb2c9f46, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/03505cf7fde640908f34e736feadee87, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e586ff8e178d4b409439ae8da7fd0eae] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=47.3 K 2024-11-12T18:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:15:59,436 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 562dc07d1fea4c35bd2a4acfe92f20ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731435354702 2024-11-12T18:15:59,436 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a36d0263004d0ebca66514fb2c9f46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731435355074 2024-11-12T18:15:59,436 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 03505cf7fde640908f34e736feadee87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731435355876 2024-11-12T18:15:59,438 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e586ff8e178d4b409439ae8da7fd0eae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731435356984 2024-11-12T18:15:59,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/78f66af8b70147b7b638399241b6ca2c is 50, key is test_row_0/A:col10/1731435359432/Put/seqid=0 2024-11-12T18:15:59,449 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:15:59,449 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/9d43c13782d942689b55503aae9da085 is 50, key is test_row_0/C:col10/1731435358100/Put/seqid=0 2024-11-12T18:15:59,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435419440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742317_1493 (size=12425) 2024-11-12T18:15:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435419456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,470 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/9d43c13782d942689b55503aae9da085 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/9d43c13782d942689b55503aae9da085 2024-11-12T18:15:59,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435419456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435419457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435419457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,480 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into 9d43c13782d942689b55503aae9da085(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:15:59,480 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:15:59,480 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=12, startTime=1731435358989; duration=0sec 2024-11-12T18:15:59,480 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:15:59,480 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:15:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742318_1494 (size=16931) 2024-11-12T18:15:59,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435419560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435419580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435419580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435419580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435419580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435419770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435419800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435419801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435419801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435419801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:15:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:15:59,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/78f66af8b70147b7b638399241b6ca2c 2024-11-12T18:15:59,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/dfc078adc8534a9bab8600cc8f6b3fd0 is 50, key is test_row_0/B:col10/1731435359432/Put/seqid=0 2024-11-12T18:15:59,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742319_1495 (size=12151) 2024-11-12T18:16:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T18:16:00,021 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:00,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T18:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T18:16:00,023 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:00,024 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:00,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435420080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T18:16:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435420120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435420121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435420121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435420121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T18:16:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/dfc078adc8534a9bab8600cc8f6b3fd0 2024-11-12T18:16:00,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/6db13cb748e345509fdeb9dbe9c6c1a7 is 50, key is test_row_0/C:col10/1731435359432/Put/seqid=0 2024-11-12T18:16:00,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742320_1496 (size=12151) 2024-11-12T18:16:00,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T18:16:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T18:16:00,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T18:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435420600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T18:16:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T18:16:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435420641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435420641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435420643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435420645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:00,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/6db13cb748e345509fdeb9dbe9c6c1a7 2024-11-12T18:16:00,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/78f66af8b70147b7b638399241b6ca2c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/78f66af8b70147b7b638399241b6ca2c 2024-11-12T18:16:00,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/78f66af8b70147b7b638399241b6ca2c, entries=250, sequenceid=192, filesize=16.5 K 2024-11-12T18:16:00,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/dfc078adc8534a9bab8600cc8f6b3fd0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/dfc078adc8534a9bab8600cc8f6b3fd0 2024-11-12T18:16:00,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/dfc078adc8534a9bab8600cc8f6b3fd0, entries=150, sequenceid=192, filesize=11.9 K 2024-11-12T18:16:00,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/6db13cb748e345509fdeb9dbe9c6c1a7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6db13cb748e345509fdeb9dbe9c6c1a7 2024-11-12T18:16:00,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6db13cb748e345509fdeb9dbe9c6c1a7, entries=150, sequenceid=192, filesize=11.9 K 2024-11-12T18:16:00,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 0f73c86ec05d2a0b97b6becb163767f7 in 1303ms, sequenceid=192, compaction requested=false 2024-11-12T18:16:00,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:00,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T18:16:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:00,794 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:16:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:00,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:00,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/c3be23f265f34e5bb9d0245d28ff95ff is 50, key is test_row_0/A:col10/1731435359453/Put/seqid=0 2024-11-12T18:16:00,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742321_1497 (size=12151) 2024-11-12T18:16:00,812 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/c3be23f265f34e5bb9d0245d28ff95ff 2024-11-12T18:16:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/cf857fdc4df942f29e3784529d60a0c8 is 50, key is test_row_0/B:col10/1731435359453/Put/seqid=0 2024-11-12T18:16:00,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742322_1498 (size=12151) 2024-11-12T18:16:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T18:16:01,241 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/cf857fdc4df942f29e3784529d60a0c8 2024-11-12T18:16:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/625a24af1eea4a5a8ec0cc9d06400152 is 50, key is test_row_0/C:col10/1731435359453/Put/seqid=0 2024-11-12T18:16:01,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742323_1499 (size=12151) 2024-11-12T18:16:01,261 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/625a24af1eea4a5a8ec0cc9d06400152 2024-11-12T18:16:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/c3be23f265f34e5bb9d0245d28ff95ff as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c3be23f265f34e5bb9d0245d28ff95ff 2024-11-12T18:16:01,272 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c3be23f265f34e5bb9d0245d28ff95ff, entries=150, sequenceid=203, filesize=11.9 K 2024-11-12T18:16:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/cf857fdc4df942f29e3784529d60a0c8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cf857fdc4df942f29e3784529d60a0c8 2024-11-12T18:16:01,277 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cf857fdc4df942f29e3784529d60a0c8, entries=150, sequenceid=203, filesize=11.9 K 2024-11-12T18:16:01,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/625a24af1eea4a5a8ec0cc9d06400152 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/625a24af1eea4a5a8ec0cc9d06400152 2024-11-12T18:16:01,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/625a24af1eea4a5a8ec0cc9d06400152, entries=150, sequenceid=203, filesize=11.9 K 2024-11-12T18:16:01,283 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 0f73c86ec05d2a0b97b6becb163767f7 in 490ms, sequenceid=203, compaction requested=true 2024-11-12T18:16:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T18:16:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T18:16:01,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T18:16:01,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2620 sec 2024-11-12T18:16:01,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.2700 sec 2024-11-12T18:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:01,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:16:01,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:01,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:01,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:01,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:01,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:01,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:01,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/5f6990078e7a4b63b023c9743bc68757 is 50, key is test_row_0/A:col10/1731435361647/Put/seqid=0 2024-11-12T18:16:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742324_1500 (size=14541) 2024-11-12T18:16:01,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/5f6990078e7a4b63b023c9743bc68757 2024-11-12T18:16:01,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/84e266a10b364ad08d2511666b2f650e is 50, key is test_row_0/B:col10/1731435361647/Put/seqid=0 2024-11-12T18:16:01,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742325_1501 (size=12151) 2024-11-12T18:16:01,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435361680, max timestamp=1731435361727Num uniques 859; 2024-11-12T18:16:01,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:16:01,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435361680, max timestamp=1731435361727Num uniques 859; 2024-11-12T18:16:01,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:16:01,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435361680, max timestamp=1731435361727Num uniques 859; 2024-11-12T18:16:01,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:16:01,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435421764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435421766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435421767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435421768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435421777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435421880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435421891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435421891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435421891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:01,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435421900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:01,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/84e266a10b364ad08d2511666b2f650e 2024-11-12T18:16:02,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/3801d24230cd4143a5ee57d32c69bf3d is 50, key is test_row_0/C:col10/1731435361647/Put/seqid=0 2024-11-12T18:16:02,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435422100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435422111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435422111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435422111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435422120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742326_1502 (size=12151) 2024-11-12T18:16:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T18:16:02,170 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:02,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T18:16:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T18:16:02,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:02,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:02,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T18:16:02,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T18:16:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:02,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435422431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435422431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435422431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435422431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435422431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T18:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T18:16:02,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/3801d24230cd4143a5ee57d32c69bf3d 2024-11-12T18:16:02,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/5f6990078e7a4b63b023c9743bc68757 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5f6990078e7a4b63b023c9743bc68757 2024-11-12T18:16:02,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5f6990078e7a4b63b023c9743bc68757, entries=200, sequenceid=213, filesize=14.2 K 2024-11-12T18:16:02,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/84e266a10b364ad08d2511666b2f650e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/84e266a10b364ad08d2511666b2f650e 2024-11-12T18:16:02,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/84e266a10b364ad08d2511666b2f650e, entries=150, sequenceid=213, filesize=11.9 K 2024-11-12T18:16:02,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/3801d24230cd4143a5ee57d32c69bf3d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/3801d24230cd4143a5ee57d32c69bf3d 2024-11-12T18:16:02,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/3801d24230cd4143a5ee57d32c69bf3d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-12T18:16:02,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0f73c86ec05d2a0b97b6becb163767f7 in 906ms, sequenceid=213, compaction requested=true 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:02,562 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:02,562 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:02,563 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:02,563 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:02,563 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:02,563 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:02,563 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,563 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,563 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/abdda80d3fac48e89909801d693b788b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/78f66af8b70147b7b638399241b6ca2c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c3be23f265f34e5bb9d0245d28ff95ff, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5f6990078e7a4b63b023c9743bc68757] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=54.7 K 2024-11-12T18:16:02,563 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/7db937547ad840f9af7fd2ab6087aeb0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/dfc078adc8534a9bab8600cc8f6b3fd0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cf857fdc4df942f29e3784529d60a0c8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/84e266a10b364ad08d2511666b2f650e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=47.7 K 2024-11-12T18:16:02,564 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting abdda80d3fac48e89909801d693b788b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731435356984 2024-11-12T18:16:02,564 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db937547ad840f9af7fd2ab6087aeb0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731435356984 2024-11-12T18:16:02,565 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f66af8b70147b7b638399241b6ca2c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731435358239 2024-11-12T18:16:02,565 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc078adc8534a9bab8600cc8f6b3fd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731435358242 2024-11-12T18:16:02,565 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3be23f265f34e5bb9d0245d28ff95ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731435359453 2024-11-12T18:16:02,565 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting cf857fdc4df942f29e3784529d60a0c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731435359453 2024-11-12T18:16:02,566 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e266a10b364ad08d2511666b2f650e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731435361621 2024-11-12T18:16:02,566 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6990078e7a4b63b023c9743bc68757, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731435361621 2024-11-12T18:16:02,592 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#425 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:02,592 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2ea915dd255a4655ba3097fcdbf8f206 is 50, key is test_row_0/A:col10/1731435361647/Put/seqid=0 2024-11-12T18:16:02,603 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:02,603 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/d16326444a8d4750bdf27f82787ac1ec is 50, key is test_row_0/B:col10/1731435361647/Put/seqid=0 2024-11-12T18:16:02,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:02,638 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:02,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:02,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742327_1503 (size=12561) 2024-11-12T18:16:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/67f31fa6ac3645929d5d914926b0b39e is 50, key is test_row_0/A:col10/1731435361775/Put/seqid=0 2024-11-12T18:16:02,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742328_1504 (size=12561) 2024-11-12T18:16:02,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742329_1505 (size=12151) 2024-11-12T18:16:02,681 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/67f31fa6ac3645929d5d914926b0b39e 2024-11-12T18:16:02,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/1501f1f041ef47b5859c358e425fd45a is 50, key is test_row_0/B:col10/1731435361775/Put/seqid=0 2024-11-12T18:16:02,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742330_1506 (size=12151) 2024-11-12T18:16:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T18:16:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:02,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435422962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435422963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435422965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435422970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435422981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,049 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2ea915dd255a4655ba3097fcdbf8f206 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2ea915dd255a4655ba3097fcdbf8f206 2024-11-12T18:16:03,055 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into 2ea915dd255a4655ba3097fcdbf8f206(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:03,055 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:03,055 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=12, startTime=1731435362562; duration=0sec 2024-11-12T18:16:03,055 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:03,055 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:03,055 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:03,058 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:03,058 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:03,058 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,058 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/9d43c13782d942689b55503aae9da085, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6db13cb748e345509fdeb9dbe9c6c1a7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/625a24af1eea4a5a8ec0cc9d06400152, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/3801d24230cd4143a5ee57d32c69bf3d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=47.7 K 2024-11-12T18:16:03,059 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d43c13782d942689b55503aae9da085, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731435356984 2024-11-12T18:16:03,059 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db13cb748e345509fdeb9dbe9c6c1a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731435358242 2024-11-12T18:16:03,060 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 625a24af1eea4a5a8ec0cc9d06400152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731435359453 2024-11-12T18:16:03,060 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3801d24230cd4143a5ee57d32c69bf3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731435361621 2024-11-12T18:16:03,062 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/d16326444a8d4750bdf27f82787ac1ec as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d16326444a8d4750bdf27f82787ac1ec 2024-11-12T18:16:03,069 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into d16326444a8d4750bdf27f82787ac1ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:03,070 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:03,070 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=12, startTime=1731435362562; duration=0sec 2024-11-12T18:16:03,070 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:03,070 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:03,070 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:03,070 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/a3cb3208946b4b67b28ff82078c2a4df is 50, key is test_row_0/C:col10/1731435361647/Put/seqid=0 2024-11-12T18:16:03,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742331_1507 (size=12561) 2024-11-12T18:16:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435423091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435423091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435423091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435423091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435423094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,112 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/1501f1f041ef47b5859c358e425fd45a 2024-11-12T18:16:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/ead9a8157e964154bf073d66421d994e is 50, key is test_row_0/C:col10/1731435361775/Put/seqid=0 2024-11-12T18:16:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742332_1508 (size=12151) 2024-11-12T18:16:03,130 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/ead9a8157e964154bf073d66421d994e 2024-11-12T18:16:03,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/67f31fa6ac3645929d5d914926b0b39e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/67f31fa6ac3645929d5d914926b0b39e 2024-11-12T18:16:03,150 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/67f31fa6ac3645929d5d914926b0b39e, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T18:16:03,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/1501f1f041ef47b5859c358e425fd45a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1501f1f041ef47b5859c358e425fd45a 2024-11-12T18:16:03,156 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1501f1f041ef47b5859c358e425fd45a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T18:16:03,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/ead9a8157e964154bf073d66421d994e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ead9a8157e964154bf073d66421d994e 2024-11-12T18:16:03,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ead9a8157e964154bf073d66421d994e, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T18:16:03,163 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 0f73c86ec05d2a0b97b6becb163767f7 in 525ms, sequenceid=248, compaction requested=false 2024-11-12T18:16:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T18:16:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T18:16:03,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T18:16:03,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 993 msec 2024-11-12T18:16:03,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 999 msec 2024-11-12T18:16:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T18:16:03,312 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:03,313 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-12T18:16:03,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T18:16:03,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:03,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:03,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:16:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:03,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/60bd93530c1e47e890041d8d9e771030 is 50, key is test_row_0/A:col10/1731435363321/Put/seqid=0 2024-11-12T18:16:03,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742333_1509 (size=12301) 2024-11-12T18:16:03,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/60bd93530c1e47e890041d8d9e771030 2024-11-12T18:16:03,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/eb0fd752217f48dfa589464d1c19264f is 50, key is test_row_0/B:col10/1731435363321/Put/seqid=0 2024-11-12T18:16:03,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742334_1510 (size=12301) 2024-11-12T18:16:03,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435423364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435423366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435423367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435423376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435423383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T18:16:03,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T18:16:03,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:03,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,491 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/a3cb3208946b4b67b28ff82078c2a4df as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a3cb3208946b4b67b28ff82078c2a4df 2024-11-12T18:16:03,496 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into a3cb3208946b4b67b28ff82078c2a4df(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:03,496 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:03,496 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=12, startTime=1731435362562; duration=0sec 2024-11-12T18:16:03,497 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:03,497 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:03,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435423490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435423490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435423490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435423501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435423501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T18:16:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:03,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T18:16:03,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435423710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435423711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435423711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435423720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435423720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:03,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/eb0fd752217f48dfa589464d1c19264f 2024-11-12T18:16:03,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/89096095cc4944a1ae629e69187308be is 50, key is test_row_0/C:col10/1731435363321/Put/seqid=0 2024-11-12T18:16:03,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742335_1511 (size=12301) 2024-11-12T18:16:03,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/89096095cc4944a1ae629e69187308be 2024-11-12T18:16:03,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/60bd93530c1e47e890041d8d9e771030 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/60bd93530c1e47e890041d8d9e771030 2024-11-12T18:16:03,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T18:16:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:03,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/60bd93530c1e47e890041d8d9e771030, entries=150, sequenceid=262, filesize=12.0 K 2024-11-12T18:16:03,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/eb0fd752217f48dfa589464d1c19264f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/eb0fd752217f48dfa589464d1c19264f 2024-11-12T18:16:03,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/eb0fd752217f48dfa589464d1c19264f, entries=150, sequenceid=262, filesize=12.0 K 2024-11-12T18:16:03,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/89096095cc4944a1ae629e69187308be as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/89096095cc4944a1ae629e69187308be 2024-11-12T18:16:03,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/89096095cc4944a1ae629e69187308be, entries=150, sequenceid=262, filesize=12.0 K 2024-11-12T18:16:03,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0f73c86ec05d2a0b97b6becb163767f7 in 482ms, sequenceid=262, compaction requested=true 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:03,804 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:03,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:03,804 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:03,805 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,805 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,805 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2ea915dd255a4655ba3097fcdbf8f206, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/67f31fa6ac3645929d5d914926b0b39e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/60bd93530c1e47e890041d8d9e771030] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.1 K 2024-11-12T18:16:03,805 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d16326444a8d4750bdf27f82787ac1ec, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1501f1f041ef47b5859c358e425fd45a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/eb0fd752217f48dfa589464d1c19264f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.1 K 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea915dd255a4655ba3097fcdbf8f206, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731435361621 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d16326444a8d4750bdf27f82787ac1ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731435361621 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f31fa6ac3645929d5d914926b0b39e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731435361758 2024-11-12T18:16:03,805 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1501f1f041ef47b5859c358e425fd45a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731435361758 2024-11-12T18:16:03,806 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0fd752217f48dfa589464d1c19264f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731435362979 2024-11-12T18:16:03,806 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60bd93530c1e47e890041d8d9e771030, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731435362979 2024-11-12T18:16:03,813 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:03,813 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/eaad418807de4054af2c048bd5a39f83 is 50, key is test_row_0/A:col10/1731435363321/Put/seqid=0 2024-11-12T18:16:03,823 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:03,823 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/89bcdb76e6d349f3833303dc4c8b8ac1 is 50, key is test_row_0/B:col10/1731435363321/Put/seqid=0 2024-11-12T18:16:03,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742336_1512 (size=12813) 2024-11-12T18:16:03,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742337_1513 (size=12813) 2024-11-12T18:16:03,842 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/89bcdb76e6d349f3833303dc4c8b8ac1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/89bcdb76e6d349f3833303dc4c8b8ac1 2024-11-12T18:16:03,855 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into 89bcdb76e6d349f3833303dc4c8b8ac1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:03,855 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:03,855 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=13, startTime=1731435363804; duration=0sec 2024-11-12T18:16:03,855 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:03,855 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:03,855 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:03,856 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:03,856 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:03,856 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,856 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a3cb3208946b4b67b28ff82078c2a4df, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ead9a8157e964154bf073d66421d994e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/89096095cc4944a1ae629e69187308be] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.1 K 2024-11-12T18:16:03,857 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a3cb3208946b4b67b28ff82078c2a4df, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731435361621 2024-11-12T18:16:03,858 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ead9a8157e964154bf073d66421d994e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731435361758 2024-11-12T18:16:03,859 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 89096095cc4944a1ae629e69187308be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731435362979 2024-11-12T18:16:03,867 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:03,867 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/4da011a66c86403aad319d935f2a2d2b is 50, key is test_row_0/C:col10/1731435363321/Put/seqid=0 2024-11-12T18:16:03,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742338_1514 (size=12813) 2024-11-12T18:16:03,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T18:16:03,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:03,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:16:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/5c8c8fab752d43c9aa6ad5ffaf567dce is 50, key is test_row_0/A:col10/1731435363375/Put/seqid=0 2024-11-12T18:16:03,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742339_1515 (size=12301) 2024-11-12T18:16:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T18:16:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:04,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:04,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435424044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435424049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435424050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435424051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435424058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435424161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435424162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435424170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435424170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435424170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,233 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/eaad418807de4054af2c048bd5a39f83 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/eaad418807de4054af2c048bd5a39f83 2024-11-12T18:16:04,237 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into eaad418807de4054af2c048bd5a39f83(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:04,237 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:04,237 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=13, startTime=1731435363804; duration=0sec 2024-11-12T18:16:04,237 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:04,237 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:04,289 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/4da011a66c86403aad319d935f2a2d2b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/4da011a66c86403aad319d935f2a2d2b 2024-11-12T18:16:04,294 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into 4da011a66c86403aad319d935f2a2d2b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:04,294 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:04,294 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=13, startTime=1731435363804; duration=0sec 2024-11-12T18:16:04,294 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:04,294 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:04,347 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/5c8c8fab752d43c9aa6ad5ffaf567dce 2024-11-12T18:16:04,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4905118144fa4444ab797e055c4ed608 is 50, key is test_row_0/B:col10/1731435363375/Put/seqid=0 2024-11-12T18:16:04,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742340_1516 (size=12301) 2024-11-12T18:16:04,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435424370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435424380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435424380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435424381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435424381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T18:16:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435424691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435424691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435424691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435424691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435424691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:04,761 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4905118144fa4444ab797e055c4ed608 2024-11-12T18:16:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/33b872df7620476aa940763acbe60917 is 50, key is test_row_0/C:col10/1731435363375/Put/seqid=0 2024-11-12T18:16:04,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742341_1517 (size=12301) 2024-11-12T18:16:05,173 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/33b872df7620476aa940763acbe60917 2024-11-12T18:16:05,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/5c8c8fab752d43c9aa6ad5ffaf567dce as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5c8c8fab752d43c9aa6ad5ffaf567dce 2024-11-12T18:16:05,180 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5c8c8fab752d43c9aa6ad5ffaf567dce, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T18:16:05,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4905118144fa4444ab797e055c4ed608 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4905118144fa4444ab797e055c4ed608 2024-11-12T18:16:05,184 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4905118144fa4444ab797e055c4ed608, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T18:16:05,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/33b872df7620476aa940763acbe60917 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/33b872df7620476aa940763acbe60917 2024-11-12T18:16:05,188 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/33b872df7620476aa940763acbe60917, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T18:16:05,189 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0f73c86ec05d2a0b97b6becb163767f7 in 1253ms, sequenceid=284, compaction requested=false 2024-11-12T18:16:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T18:16:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-12T18:16:05,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T18:16:05,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-11-12T18:16:05,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8810 sec 2024-11-12T18:16:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:05,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:16:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:05,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/f34c0cf9bd874c19b4407d795693f75b is 50, key is test_row_0/A:col10/1731435365213/Put/seqid=0 2024-11-12T18:16:05,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742342_1518 (size=14741) 2024-11-12T18:16:05,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435425239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435425247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435425248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435425250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435425251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435425360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435425370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435425370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435425370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435425371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T18:16:05,471 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:05,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-12T18:16:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T18:16:05,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:05,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:05,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:05,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435425570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T18:16:05,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435425581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435425590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435425590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435425590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/f34c0cf9bd874c19b4407d795693f75b 2024-11-12T18:16:05,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T18:16:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:05,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/c6de3a448e86459eb29242e7db654e8d is 50, key is test_row_0/B:col10/1731435365213/Put/seqid=0 2024-11-12T18:16:05,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742343_1519 (size=12301) 2024-11-12T18:16:05,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/c6de3a448e86459eb29242e7db654e8d 2024-11-12T18:16:05,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/49d5e3a91fca4befbb27f881f828b459 is 50, key is test_row_0/C:col10/1731435365213/Put/seqid=0 2024-11-12T18:16:05,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742344_1520 (size=12301) 2024-11-12T18:16:05,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/49d5e3a91fca4befbb27f881f828b459 2024-11-12T18:16:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/f34c0cf9bd874c19b4407d795693f75b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/f34c0cf9bd874c19b4407d795693f75b 2024-11-12T18:16:05,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/f34c0cf9bd874c19b4407d795693f75b, entries=200, sequenceid=299, filesize=14.4 K 2024-11-12T18:16:05,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/c6de3a448e86459eb29242e7db654e8d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/c6de3a448e86459eb29242e7db654e8d 2024-11-12T18:16:05,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/c6de3a448e86459eb29242e7db654e8d, entries=150, sequenceid=299, filesize=12.0 K 2024-11-12T18:16:05,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/49d5e3a91fca4befbb27f881f828b459 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/49d5e3a91fca4befbb27f881f828b459 2024-11-12T18:16:05,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/49d5e3a91fca4befbb27f881f828b459, entries=150, sequenceid=299, filesize=12.0 K 2024-11-12T18:16:05,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0f73c86ec05d2a0b97b6becb163767f7 in 483ms, sequenceid=299, compaction requested=true 2024-11-12T18:16:05,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:05,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:05,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:05,698 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:05,698 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:05,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:05,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:05,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:05,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:05,699 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:05,700 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:05,700 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,700 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/89bcdb76e6d349f3833303dc4c8b8ac1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4905118144fa4444ab797e055c4ed608, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/c6de3a448e86459eb29242e7db654e8d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.5 K 2024-11-12T18:16:05,700 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:05,700 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:05,700 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,700 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/eaad418807de4054af2c048bd5a39f83, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5c8c8fab752d43c9aa6ad5ffaf567dce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/f34c0cf9bd874c19b4407d795693f75b] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=38.9 K 2024-11-12T18:16:05,701 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 89bcdb76e6d349f3833303dc4c8b8ac1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731435362979 2024-11-12T18:16:05,701 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaad418807de4054af2c048bd5a39f83, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731435362979 2024-11-12T18:16:05,703 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4905118144fa4444ab797e055c4ed608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731435363363 2024-11-12T18:16:05,703 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c8c8fab752d43c9aa6ad5ffaf567dce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731435363363 2024-11-12T18:16:05,703 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting c6de3a448e86459eb29242e7db654e8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731435364047 2024-11-12T18:16:05,703 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34c0cf9bd874c19b4407d795693f75b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731435364047 2024-11-12T18:16:05,711 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:05,712 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ec411a29611843a2b03a453b026c5b83 is 50, key is test_row_0/B:col10/1731435365213/Put/seqid=0 2024-11-12T18:16:05,714 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:05,714 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/57e6061f3dae4b79b4b908f3f0c9b8ab is 50, key is test_row_0/A:col10/1731435365213/Put/seqid=0 2024-11-12T18:16:05,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742345_1521 (size=12915) 2024-11-12T18:16:05,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742346_1522 (size=12915) 2024-11-12T18:16:05,725 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ec411a29611843a2b03a453b026c5b83 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ec411a29611843a2b03a453b026c5b83 2024-11-12T18:16:05,729 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into ec411a29611843a2b03a453b026c5b83(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:05,729 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:05,729 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=13, startTime=1731435365698; duration=0sec 2024-11-12T18:16:05,729 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:05,729 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:05,729 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:05,731 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:05,731 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:05,731 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,731 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/4da011a66c86403aad319d935f2a2d2b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/33b872df7620476aa940763acbe60917, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/49d5e3a91fca4befbb27f881f828b459] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.5 K 2024-11-12T18:16:05,731 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da011a66c86403aad319d935f2a2d2b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731435362979 2024-11-12T18:16:05,732 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b872df7620476aa940763acbe60917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731435363363 2024-11-12T18:16:05,732 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 49d5e3a91fca4befbb27f881f828b459, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731435364047 2024-11-12T18:16:05,738 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:05,738 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/04fa882b84094abcb7529792245eee6f is 50, key is test_row_0/C:col10/1731435365213/Put/seqid=0 2024-11-12T18:16:05,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742347_1523 (size=12915) 2024-11-12T18:16:05,746 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/04fa882b84094abcb7529792245eee6f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/04fa882b84094abcb7529792245eee6f 2024-11-12T18:16:05,751 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into 04fa882b84094abcb7529792245eee6f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:05,751 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:05,751 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=13, startTime=1731435365698; duration=0sec 2024-11-12T18:16:05,751 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:05,751 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:05,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T18:16:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:05,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:16:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:05,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/d8fbf85d47bb4308b7ad24a58d96870f is 50, key is test_row_0/A:col10/1731435365246/Put/seqid=0 2024-11-12T18:16:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T18:16:05,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742348_1524 (size=12301) 2024-11-12T18:16:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:05,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:05,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435425910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435425913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435425918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435425921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435425922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435426030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435426030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435426040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435426040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T18:16:06,126 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/57e6061f3dae4b79b4b908f3f0c9b8ab as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/57e6061f3dae4b79b4b908f3f0c9b8ab 2024-11-12T18:16:06,131 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into 57e6061f3dae4b79b4b908f3f0c9b8ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:06,131 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:06,131 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=13, startTime=1731435365698; duration=0sec 2024-11-12T18:16:06,131 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:06,131 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:06,198 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/d8fbf85d47bb4308b7ad24a58d96870f 2024-11-12T18:16:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/cd986ca0cb484a0fab309ad42b1fb0bb is 50, key is test_row_0/B:col10/1731435365246/Put/seqid=0 2024-11-12T18:16:06,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742349_1525 (size=12301) 2024-11-12T18:16:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435426240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435426251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435426261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435426261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435426430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435426550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435426570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435426570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435426572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:06,613 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/cd986ca0cb484a0fab309ad42b1fb0bb 2024-11-12T18:16:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T18:16:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/320d5f86f54649a39e1c1283fdeb3b3b is 50, key is test_row_0/C:col10/1731435365246/Put/seqid=0 2024-11-12T18:16:06,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742350_1526 (size=12301) 2024-11-12T18:16:06,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/320d5f86f54649a39e1c1283fdeb3b3b 2024-11-12T18:16:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/d8fbf85d47bb4308b7ad24a58d96870f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/d8fbf85d47bb4308b7ad24a58d96870f 2024-11-12T18:16:06,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/d8fbf85d47bb4308b7ad24a58d96870f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T18:16:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/cd986ca0cb484a0fab309ad42b1fb0bb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cd986ca0cb484a0fab309ad42b1fb0bb 2024-11-12T18:16:06,663 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cd986ca0cb484a0fab309ad42b1fb0bb, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T18:16:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/320d5f86f54649a39e1c1283fdeb3b3b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/320d5f86f54649a39e1c1283fdeb3b3b 2024-11-12T18:16:06,672 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/320d5f86f54649a39e1c1283fdeb3b3b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T18:16:06,673 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0f73c86ec05d2a0b97b6becb163767f7 in 893ms, sequenceid=320, compaction requested=false 2024-11-12T18:16:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-12T18:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-12T18:16:06,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T18:16:06,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2010 sec 2024-11-12T18:16:06,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.2090 sec 2024-11-12T18:16:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:07,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:07,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/e056a96460b24650b4fe691d5c5382f0 is 50, key is test_row_0/A:col10/1731435365911/Put/seqid=0 2024-11-12T18:16:07,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742351_1527 (size=14741) 2024-11-12T18:16:07,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/e056a96460b24650b4fe691d5c5382f0 2024-11-12T18:16:07,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/d0cfe03e83fc417d97f5177cbed45bce is 50, key is test_row_0/B:col10/1731435365911/Put/seqid=0 2024-11-12T18:16:07,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742352_1528 (size=12301) 2024-11-12T18:16:07,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/d0cfe03e83fc417d97f5177cbed45bce 2024-11-12T18:16:07,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435427125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435427127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435427134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/d437e56cd334498c8b7d9627b32aaf31 is 50, key is test_row_0/C:col10/1731435365911/Put/seqid=0 2024-11-12T18:16:07,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435427140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742353_1529 (size=12301) 2024-11-12T18:16:07,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435427261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435427261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435427261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435427261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435427450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,456 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in 0f73c86ec05d2a0b97b6becb163767f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435427480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435427480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435427480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435427481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/d437e56cd334498c8b7d9627b32aaf31 2024-11-12T18:16:07,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/e056a96460b24650b4fe691d5c5382f0 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/e056a96460b24650b4fe691d5c5382f0 2024-11-12T18:16:07,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/e056a96460b24650b4fe691d5c5382f0, entries=200, sequenceid=335, filesize=14.4 K 2024-11-12T18:16:07,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/d0cfe03e83fc417d97f5177cbed45bce as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d0cfe03e83fc417d97f5177cbed45bce 2024-11-12T18:16:07,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d0cfe03e83fc417d97f5177cbed45bce, entries=150, sequenceid=335, filesize=12.0 K 2024-11-12T18:16:07,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/d437e56cd334498c8b7d9627b32aaf31 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/d437e56cd334498c8b7d9627b32aaf31 2024-11-12T18:16:07,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/d437e56cd334498c8b7d9627b32aaf31, entries=150, sequenceid=335, filesize=12.0 K 2024-11-12T18:16:07,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0f73c86ec05d2a0b97b6becb163767f7 in 499ms, sequenceid=335, compaction requested=true 2024-11-12T18:16:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:07,577 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:07,577 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:07,577 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:07,577 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:07,577 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:07,578 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:07,578 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:07,578 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:07,578 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/57e6061f3dae4b79b4b908f3f0c9b8ab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/d8fbf85d47bb4308b7ad24a58d96870f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/e056a96460b24650b4fe691d5c5382f0] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=39.0 K 2024-11-12T18:16:07,578 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ec411a29611843a2b03a453b026c5b83, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cd986ca0cb484a0fab309ad42b1fb0bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d0cfe03e83fc417d97f5177cbed45bce] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.6 K 2024-11-12T18:16:07,578 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e6061f3dae4b79b4b908f3f0c9b8ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731435364047 2024-11-12T18:16:07,578 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ec411a29611843a2b03a453b026c5b83, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731435364047 2024-11-12T18:16:07,578 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fbf85d47bb4308b7ad24a58d96870f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731435365246 2024-11-12T18:16:07,578 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting cd986ca0cb484a0fab309ad42b1fb0bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731435365246 2024-11-12T18:16:07,579 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e056a96460b24650b4fe691d5c5382f0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731435365911 2024-11-12T18:16:07,579 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cfe03e83fc417d97f5177cbed45bce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731435365911 2024-11-12T18:16:07,588 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:07,589 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/a47307a3b6934f758471c1230e7cb53e is 50, key is test_row_0/A:col10/1731435365911/Put/seqid=0 2024-11-12T18:16:07,591 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:07,592 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4ed26b795fff4cd9bb86b280e8975580 is 50, key is test_row_0/B:col10/1731435365911/Put/seqid=0 2024-11-12T18:16:07,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742355_1531 (size=13017) 2024-11-12T18:16:07,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742354_1530 (size=13017) 2024-11-12T18:16:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T18:16:07,621 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:07,624 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-12T18:16:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T18:16:07,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:07,627 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:07,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T18:16:07,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:07,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2e674194c99a43caba7007ca1e402b64 is 50, key is test_row_0/A:col10/1731435367121/Put/seqid=0 2024-11-12T18:16:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:07,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742356_1532 (size=12301) 2024-11-12T18:16:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435427862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435427863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435427885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435427885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T18:16:08,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435427990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435427992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,019 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/4ed26b795fff4cd9bb86b280e8975580 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4ed26b795fff4cd9bb86b280e8975580 2024-11-12T18:16:08,019 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/a47307a3b6934f758471c1230e7cb53e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/a47307a3b6934f758471c1230e7cb53e 2024-11-12T18:16:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435428010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435428010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,029 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into a47307a3b6934f758471c1230e7cb53e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:08,029 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:08,029 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=13, startTime=1731435367576; duration=0sec 2024-11-12T18:16:08,029 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into 4ed26b795fff4cd9bb86b280e8975580(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:08,029 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:08,029 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:08,029 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:08,031 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:08,031 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=13, startTime=1731435367576; duration=0sec 2024-11-12T18:16:08,032 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:08,032 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:08,032 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:08,032 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:08,032 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:08,032 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/04fa882b84094abcb7529792245eee6f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/320d5f86f54649a39e1c1283fdeb3b3b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/d437e56cd334498c8b7d9627b32aaf31] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.6 K 2024-11-12T18:16:08,033 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04fa882b84094abcb7529792245eee6f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731435364047 2024-11-12T18:16:08,033 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 320d5f86f54649a39e1c1283fdeb3b3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731435365246 2024-11-12T18:16:08,034 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting d437e56cd334498c8b7d9627b32aaf31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731435365911 2024-11-12T18:16:08,043 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:08,044 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/62f6488a6da244b1ab7a4af27b721f2b is 50, key is test_row_0/C:col10/1731435365911/Put/seqid=0 2024-11-12T18:16:08,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742357_1533 (size=13017) 2024-11-12T18:16:08,064 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/62f6488a6da244b1ab7a4af27b721f2b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/62f6488a6da244b1ab7a4af27b721f2b 2024-11-12T18:16:08,068 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into 62f6488a6da244b1ab7a4af27b721f2b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:08,068 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:08,069 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=13, startTime=1731435367577; duration=0sec 2024-11-12T18:16:08,069 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:08,069 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435428210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435428210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,221 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2e674194c99a43caba7007ca1e402b64 2024-11-12T18:16:08,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/3e98cbc46b32401aad9f043bdd1cda86 is 50, key is test_row_0/B:col10/1731435367121/Put/seqid=0 2024-11-12T18:16:08,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435428233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435428233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T18:16:08,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742358_1534 (size=12301) 2024-11-12T18:16:08,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435428536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435428536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435428553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435428554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:08,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/3e98cbc46b32401aad9f043bdd1cda86 2024-11-12T18:16:08,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/b81d8df69d41440bacc8698d0b718395 is 50, key is test_row_0/C:col10/1731435367121/Put/seqid=0 2024-11-12T18:16:08,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742359_1535 (size=12301) 2024-11-12T18:16:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T18:16:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435429060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:09,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435429060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:09,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435429070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435429080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:09,121 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/b81d8df69d41440bacc8698d0b718395 2024-11-12T18:16:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2e674194c99a43caba7007ca1e402b64 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e674194c99a43caba7007ca1e402b64 2024-11-12T18:16:09,129 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e674194c99a43caba7007ca1e402b64, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T18:16:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/3e98cbc46b32401aad9f043bdd1cda86 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/3e98cbc46b32401aad9f043bdd1cda86 2024-11-12T18:16:09,140 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/3e98cbc46b32401aad9f043bdd1cda86, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T18:16:09,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/b81d8df69d41440bacc8698d0b718395 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/b81d8df69d41440bacc8698d0b718395 2024-11-12T18:16:09,146 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/b81d8df69d41440bacc8698d0b718395, entries=150, sequenceid=353, filesize=12.0 K 2024-11-12T18:16:09,148 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0f73c86ec05d2a0b97b6becb163767f7 in 1367ms, sequenceid=353, compaction requested=false 2024-11-12T18:16:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-12T18:16:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-12T18:16:09,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-12T18:16:09,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-11-12T18:16:09,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5280 sec 2024-11-12T18:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:09,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:16:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:09,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:09,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/803d3b1f1fba4c29b12f4e9fbc8a6833 is 50, key is test_row_0/A:col10/1731435367827/Put/seqid=0 2024-11-12T18:16:09,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742360_1536 (size=14741) 2024-11-12T18:16:09,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/803d3b1f1fba4c29b12f4e9fbc8a6833 2024-11-12T18:16:09,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/1683e19e5e3d477180974566b56c4cc6 is 50, key is test_row_0/B:col10/1731435367827/Put/seqid=0 2024-11-12T18:16:09,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742361_1537 (size=12301) 2024-11-12T18:16:09,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/1683e19e5e3d477180974566b56c4cc6 2024-11-12T18:16:09,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/1d4f3f473f534d0a95dca21a066d224a is 50, key is test_row_0/C:col10/1731435367827/Put/seqid=0 2024-11-12T18:16:09,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742362_1538 (size=12301) 2024-11-12T18:16:09,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435429616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:09,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435429740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T18:16:09,771 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:09,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-12T18:16:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T18:16:09,773 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:09,774 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:09,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T18:16:09,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/1d4f3f473f534d0a95dca21a066d224a 2024-11-12T18:16:09,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/803d3b1f1fba4c29b12f4e9fbc8a6833 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/803d3b1f1fba4c29b12f4e9fbc8a6833 2024-11-12T18:16:09,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/803d3b1f1fba4c29b12f4e9fbc8a6833, entries=200, sequenceid=371, filesize=14.4 K 2024-11-12T18:16:09,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/1683e19e5e3d477180974566b56c4cc6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1683e19e5e3d477180974566b56c4cc6 2024-11-12T18:16:09,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:09,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:09,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1683e19e5e3d477180974566b56c4cc6, entries=150, sequenceid=371, filesize=12.0 K 2024-11-12T18:16:09,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/1d4f3f473f534d0a95dca21a066d224a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1d4f3f473f534d0a95dca21a066d224a 2024-11-12T18:16:09,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1d4f3f473f534d0a95dca21a066d224a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-12T18:16:09,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0f73c86ec05d2a0b97b6becb163767f7 in 464ms, sequenceid=371, compaction requested=true 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:09,941 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:09,941 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:09,942 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:09,942 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:09,942 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:09,947 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:09,947 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/a47307a3b6934f758471c1230e7cb53e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e674194c99a43caba7007ca1e402b64, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/803d3b1f1fba4c29b12f4e9fbc8a6833] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=39.1 K 2024-11-12T18:16:09,947 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:09,947 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:09,948 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4ed26b795fff4cd9bb86b280e8975580, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/3e98cbc46b32401aad9f043bdd1cda86, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1683e19e5e3d477180974566b56c4cc6] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.7 K 2024-11-12T18:16:09,948 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed26b795fff4cd9bb86b280e8975580, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731435365911 2024-11-12T18:16:09,948 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting a47307a3b6934f758471c1230e7cb53e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731435365911 2024-11-12T18:16:09,949 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e98cbc46b32401aad9f043bdd1cda86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731435367121 2024-11-12T18:16:09,949 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e674194c99a43caba7007ca1e402b64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731435367121 2024-11-12T18:16:09,949 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1683e19e5e3d477180974566b56c4cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731435367827 2024-11-12T18:16:09,950 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803d3b1f1fba4c29b12f4e9fbc8a6833, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731435367827 2024-11-12T18:16:09,961 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:09,962 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ffdf0e830e8b42aabcf50a440c1053ad is 50, key is test_row_0/B:col10/1731435367827/Put/seqid=0 2024-11-12T18:16:09,963 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:09,963 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/8896ff0b967f4e7cb1e1acf010325068 is 50, key is test_row_0/A:col10/1731435367827/Put/seqid=0 2024-11-12T18:16:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:09,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:16:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:09,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:09,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742363_1539 (size=13119) 2024-11-12T18:16:09,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/69478f4b13574a3bb8369250981d08f3 is 50, key is test_row_0/A:col10/1731435369592/Put/seqid=0 2024-11-12T18:16:09,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742364_1540 (size=13119) 2024-11-12T18:16:10,005 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ffdf0e830e8b42aabcf50a440c1053ad as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ffdf0e830e8b42aabcf50a440c1053ad 2024-11-12T18:16:10,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742365_1541 (size=14741) 2024-11-12T18:16:10,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/69478f4b13574a3bb8369250981d08f3 2024-11-12T18:16:10,012 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into ffdf0e830e8b42aabcf50a440c1053ad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:10,013 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:10,013 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=13, startTime=1731435369941; duration=0sec 2024-11-12T18:16:10,013 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:10,013 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:10,013 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:10,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/88ac1774713341b49008d72b431cc9a6 is 50, key is test_row_0/B:col10/1731435369592/Put/seqid=0 2024-11-12T18:16:10,029 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:10,029 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:10,030 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,030 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/62f6488a6da244b1ab7a4af27b721f2b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/b81d8df69d41440bacc8698d0b718395, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1d4f3f473f534d0a95dca21a066d224a] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=36.7 K 2024-11-12T18:16:10,035 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f6488a6da244b1ab7a4af27b721f2b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731435365911 2024-11-12T18:16:10,036 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b81d8df69d41440bacc8698d0b718395, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731435367121 2024-11-12T18:16:10,036 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4f3f473f534d0a95dca21a066d224a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731435367827 2024-11-12T18:16:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742366_1542 (size=12301) 2024-11-12T18:16:10,066 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:10,067 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/e8b794a2b11d4d00ad941ef0a6ae1da2 is 50, key is test_row_0/C:col10/1731435367827/Put/seqid=0 2024-11-12T18:16:10,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T18:16:10,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742367_1543 (size=13119) 2024-11-12T18:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435430085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435430091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,100 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in 0f73c86ec05d2a0b97b6becb163767f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435430097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435430097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435430100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,113 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in 0f73c86ec05d2a0b97b6becb163767f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435430210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435430220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435430221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:10,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:10,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,390 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/8896ff0b967f4e7cb1e1acf010325068 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8896ff0b967f4e7cb1e1acf010325068 2024-11-12T18:16:10,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:10,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:10,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,395 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into 8896ff0b967f4e7cb1e1acf010325068(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:10,395 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:10,395 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=13, startTime=1731435369941; duration=0sec 2024-11-12T18:16:10,395 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:10,395 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T18:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435430430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435430440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435430440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/88ac1774713341b49008d72b431cc9a6 2024-11-12T18:16:10,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/ab13b0821b16435bb6141ef6b7efdb11 is 50, key is test_row_0/C:col10/1731435369592/Put/seqid=0 2024-11-12T18:16:10,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742368_1544 (size=12301) 2024-11-12T18:16:10,504 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/e8b794a2b11d4d00ad941ef0a6ae1da2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e8b794a2b11d4d00ad941ef0a6ae1da2 2024-11-12T18:16:10,509 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into e8b794a2b11d4d00ad941ef0a6ae1da2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:10,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:10,509 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=13, startTime=1731435369941; duration=0sec 2024-11-12T18:16:10,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:10,509 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:10,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:10,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435430750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435430760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435430760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:10,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:10,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:10,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:10,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/ab13b0821b16435bb6141ef6b7efdb11 2024-11-12T18:16:10,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/69478f4b13574a3bb8369250981d08f3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/69478f4b13574a3bb8369250981d08f3 2024-11-12T18:16:10,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/69478f4b13574a3bb8369250981d08f3, entries=200, sequenceid=389, filesize=14.4 K 2024-11-12T18:16:10,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/88ac1774713341b49008d72b431cc9a6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/88ac1774713341b49008d72b431cc9a6 2024-11-12T18:16:10,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/88ac1774713341b49008d72b431cc9a6, entries=150, sequenceid=389, filesize=12.0 K 2024-11-12T18:16:10,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/ab13b0821b16435bb6141ef6b7efdb11 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ab13b0821b16435bb6141ef6b7efdb11 2024-11-12T18:16:10,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ab13b0821b16435bb6141ef6b7efdb11, entries=150, sequenceid=389, filesize=12.0 K 2024-11-12T18:16:10,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0f73c86ec05d2a0b97b6becb163767f7 in 925ms, sequenceid=389, compaction requested=false 2024-11-12T18:16:10,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T18:16:11,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-12T18:16:11,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:11,012 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:16:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:11,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:11,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2e1ac269edb54f12a0b3864c9c612f41 is 50, key is test_row_0/A:col10/1731435370076/Put/seqid=0 2024-11-12T18:16:11,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742369_1545 (size=12301) 2024-11-12T18:16:11,023 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2e1ac269edb54f12a0b3864c9c612f41 2024-11-12T18:16:11,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/5184325efea74d80859e1567768b2b72 is 50, key is test_row_0/B:col10/1731435370076/Put/seqid=0 2024-11-12T18:16:11,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742370_1546 (size=12301) 2024-11-12T18:16:11,058 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/5184325efea74d80859e1567768b2b72 2024-11-12T18:16:11,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/7e13015256204876be0515756fdf40d9 is 50, key is test_row_0/C:col10/1731435370076/Put/seqid=0 2024-11-12T18:16:11,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742371_1547 (size=12301) 2024-11-12T18:16:11,101 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/7e13015256204876be0515756fdf40d9 2024-11-12T18:16:11,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/2e1ac269edb54f12a0b3864c9c612f41 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e1ac269edb54f12a0b3864c9c612f41 2024-11-12T18:16:11,111 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e1ac269edb54f12a0b3864c9c612f41, entries=150, sequenceid=406, filesize=12.0 K 2024-11-12T18:16:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/5184325efea74d80859e1567768b2b72 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/5184325efea74d80859e1567768b2b72 2024-11-12T18:16:11,119 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/5184325efea74d80859e1567768b2b72, entries=150, sequenceid=406, filesize=12.0 K 2024-11-12T18:16:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/7e13015256204876be0515756fdf40d9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/7e13015256204876be0515756fdf40d9 2024-11-12T18:16:11,125 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/7e13015256204876be0515756fdf40d9, entries=150, sequenceid=406, filesize=12.0 K 2024-11-12T18:16:11,126 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 0f73c86ec05d2a0b97b6becb163767f7 in 114ms, sequenceid=406, compaction requested=true 2024-11-12T18:16:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-12T18:16:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-12T18:16:11,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T18:16:11,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3540 sec 2024-11-12T18:16:11,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3590 sec 2024-11-12T18:16:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:11,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:16:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:11,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/673da9f0c853441494dad9642f542713 is 50, key is test_row_0/A:col10/1731435371274/Put/seqid=0 2024-11-12T18:16:11,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742372_1548 (size=14737) 2024-11-12T18:16:11,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/673da9f0c853441494dad9642f542713 2024-11-12T18:16:11,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/a35c69c3bfa74a65a930a42ab6d6288e is 50, key is test_row_0/B:col10/1731435371274/Put/seqid=0 2024-11-12T18:16:11,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742373_1549 (size=9857) 2024-11-12T18:16:11,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435371325, max timestamp=1731435371347Num uniques 859; 2024-11-12T18:16:11,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731435371325, max timestamp=1731435371347Num uniques 859; 2024-11-12T18:16:11,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:16:11,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:16:11,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435371325, max timestamp=1731435371347Num uniques 859; 2024-11-12T18:16:11,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:16:11,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435431405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435431423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435431424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435431530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435431550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435431551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/a35c69c3bfa74a65a930a42ab6d6288e 2024-11-12T18:16:11,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/75b2658e430141a5a0084ac2c6069951 is 50, key is test_row_0/C:col10/1731435371274/Put/seqid=0 2024-11-12T18:16:11,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742374_1550 (size=9857) 2024-11-12T18:16:11,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/75b2658e430141a5a0084ac2c6069951 2024-11-12T18:16:11,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/673da9f0c853441494dad9642f542713 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/673da9f0c853441494dad9642f542713 2024-11-12T18:16:11,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/673da9f0c853441494dad9642f542713, entries=200, sequenceid=416, filesize=14.4 K 2024-11-12T18:16:11,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/a35c69c3bfa74a65a930a42ab6d6288e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/a35c69c3bfa74a65a930a42ab6d6288e 2024-11-12T18:16:11,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435431752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435431770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/a35c69c3bfa74a65a930a42ab6d6288e, entries=100, sequenceid=416, filesize=9.6 K 2024-11-12T18:16:11,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/75b2658e430141a5a0084ac2c6069951 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/75b2658e430141a5a0084ac2c6069951 2024-11-12T18:16:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435431772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:11,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/75b2658e430141a5a0084ac2c6069951, entries=100, sequenceid=416, filesize=9.6 K 2024-11-12T18:16:11,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 0f73c86ec05d2a0b97b6becb163767f7 in 509ms, sequenceid=416, compaction requested=true 2024-11-12T18:16:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:11,793 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:11,794 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:11,794 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:11,794 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:11,795 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8896ff0b967f4e7cb1e1acf010325068, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/69478f4b13574a3bb8369250981d08f3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e1ac269edb54f12a0b3864c9c612f41, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/673da9f0c853441494dad9642f542713] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=53.6 K 2024-11-12T18:16:11,795 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8896ff0b967f4e7cb1e1acf010325068, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731435367827 2024-11-12T18:16:11,795 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69478f4b13574a3bb8369250981d08f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731435369576 2024-11-12T18:16:11,796 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e1ac269edb54f12a0b3864c9c612f41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731435370076 2024-11-12T18:16:11,796 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673da9f0c853441494dad9642f542713, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731435371274 2024-11-12T18:16:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:11,801 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:11,806 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:11,806 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:11,806 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:11,806 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ffdf0e830e8b42aabcf50a440c1053ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/88ac1774713341b49008d72b431cc9a6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/5184325efea74d80859e1567768b2b72, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/a35c69c3bfa74a65a930a42ab6d6288e] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=46.5 K 2024-11-12T18:16:11,806 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdf0e830e8b42aabcf50a440c1053ad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731435367827 2024-11-12T18:16:11,807 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ac1774713341b49008d72b431cc9a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731435369576 2024-11-12T18:16:11,807 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5184325efea74d80859e1567768b2b72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731435370076 2024-11-12T18:16:11,808 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a35c69c3bfa74a65a930a42ab6d6288e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731435371274 2024-11-12T18:16:11,823 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:11,824 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/3aa5b182cc5f423d9686902a6665a863 is 50, key is test_row_0/A:col10/1731435371274/Put/seqid=0 2024-11-12T18:16:11,827 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:11,828 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/74ade584ea814cf2b48ba3ffad3aa32e is 50, key is test_row_0/B:col10/1731435371274/Put/seqid=0 2024-11-12T18:16:11,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742376_1552 (size=13255) 2024-11-12T18:16:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742375_1551 (size=13255) 2024-11-12T18:16:11,881 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/74ade584ea814cf2b48ba3ffad3aa32e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/74ade584ea814cf2b48ba3ffad3aa32e 2024-11-12T18:16:11,897 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into 74ade584ea814cf2b48ba3ffad3aa32e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:11,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:11,897 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=12, startTime=1731435371800; duration=0sec 2024-11-12T18:16:11,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:11,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:11,897 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:11,898 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/3aa5b182cc5f423d9686902a6665a863 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/3aa5b182cc5f423d9686902a6665a863 2024-11-12T18:16:11,908 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:11,908 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:11,908 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:11,908 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e8b794a2b11d4d00ad941ef0a6ae1da2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ab13b0821b16435bb6141ef6b7efdb11, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/7e13015256204876be0515756fdf40d9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/75b2658e430141a5a0084ac2c6069951] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=46.5 K 2024-11-12T18:16:11,908 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b794a2b11d4d00ad941ef0a6ae1da2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731435367827 2024-11-12T18:16:11,913 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ab13b0821b16435bb6141ef6b7efdb11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731435369576 2024-11-12T18:16:11,916 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e13015256204876be0515756fdf40d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731435370076 2024-11-12T18:16:11,917 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b2658e430141a5a0084ac2c6069951, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731435371274 2024-11-12T18:16:11,919 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into 3aa5b182cc5f423d9686902a6665a863(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:11,919 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:11,919 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=12, startTime=1731435371792; duration=0sec 2024-11-12T18:16:11,919 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:11,919 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-12T18:16:11,924 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:11,925 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:11,931 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:11,931 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/71a9c324e4124fd29af9b1a8437a7072 is 50, key is test_row_0/C:col10/1731435371274/Put/seqid=0 2024-11-12T18:16:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-12T18:16:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T18:16:11,936 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:11,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:11,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742377_1553 (size=13255) 2024-11-12T18:16:11,972 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/71a9c324e4124fd29af9b1a8437a7072 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/71a9c324e4124fd29af9b1a8437a7072 2024-11-12T18:16:11,979 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into 71a9c324e4124fd29af9b1a8437a7072(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:11,979 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:11,979 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=12, startTime=1731435371801; duration=0sec 2024-11-12T18:16:11,979 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:11,979 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T18:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:12,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-12T18:16:12,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-12T18:16:12,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:12,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:12,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:12,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:12,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/81f63d27b8da43c1ad7fd98f7f982cbc is 50, key is test_row_0/A:col10/1731435371420/Put/seqid=0 2024-11-12T18:16:12,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742378_1554 (size=14741) 2024-11-12T18:16:12,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435432116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435432117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435432138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435432138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52550 deadline: 1731435432141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,172 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in 0f73c86ec05d2a0b97b6becb163767f7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T18:16:12,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435432245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-12T18:16:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435432245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435432280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435432281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-12T18:16:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435432460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435432470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435432500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435432500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/81f63d27b8da43c1ad7fd98f7f982cbc 2024-11-12T18:16:12,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/69ec6ba6f7d14ec792f26a95f31b2a17 is 50, key is test_row_0/B:col10/1731435371420/Put/seqid=0 2024-11-12T18:16:12,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742379_1555 (size=12301) 2024-11-12T18:16:12,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/69ec6ba6f7d14ec792f26a95f31b2a17 2024-11-12T18:16:12,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/aac81db63ac14c0899ab8ada4f59bdce is 50, key is test_row_0/C:col10/1731435371420/Put/seqid=0 2024-11-12T18:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T18:16:12,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-12T18:16:12,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:12,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742380_1556 (size=12301) 2024-11-12T18:16:12,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/aac81db63ac14c0899ab8ada4f59bdce 2024-11-12T18:16:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/81f63d27b8da43c1ad7fd98f7f982cbc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/81f63d27b8da43c1ad7fd98f7f982cbc 2024-11-12T18:16:12,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/81f63d27b8da43c1ad7fd98f7f982cbc, entries=200, sequenceid=456, filesize=14.4 K 2024-11-12T18:16:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/69ec6ba6f7d14ec792f26a95f31b2a17 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/69ec6ba6f7d14ec792f26a95f31b2a17 2024-11-12T18:16:12,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/69ec6ba6f7d14ec792f26a95f31b2a17, entries=150, sequenceid=456, filesize=12.0 K 2024-11-12T18:16:12,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/aac81db63ac14c0899ab8ada4f59bdce as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/aac81db63ac14c0899ab8ada4f59bdce 2024-11-12T18:16:12,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/aac81db63ac14c0899ab8ada4f59bdce, entries=150, sequenceid=456, filesize=12.0 K 2024-11-12T18:16:12,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=46.96 KB/48090 for 0f73c86ec05d2a0b97b6becb163767f7 in 526ms, sequenceid=456, compaction requested=false 2024-11-12T18:16:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:12,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-12T18:16:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:12,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:16:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:12,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:12,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/c72bfe96585845808437db88945de8a2 is 50, key is test_row_0/A:col10/1731435372109/Put/seqid=0 2024-11-12T18:16:12,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742381_1557 (size=12301) 2024-11-12T18:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:12,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. as already flushing 2024-11-12T18:16:12,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435372809, max timestamp=1731435372871Num uniques 859; 2024-11-12T18:16:12,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:16:12,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435372809, max timestamp=1731435372871Num uniques 859; 2024-11-12T18:16:12,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:16:12,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731435372809, max timestamp=1731435372871Num uniques 859; 2024-11-12T18:16:12,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T18:16:12,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435432919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435432928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435432930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435432930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435433041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435433041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435433051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435433051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T18:16:13,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/c72bfe96585845808437db88945de8a2 2024-11-12T18:16:13,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/300a8d0f6e5a4f9a8a81f24fa3ded03b is 50, key is test_row_0/B:col10/1731435372109/Put/seqid=0 2024-11-12T18:16:13,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742382_1558 (size=12301) 2024-11-12T18:16:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435433260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435433260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435433271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435433271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,558 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/300a8d0f6e5a4f9a8a81f24fa3ded03b 2024-11-12T18:16:13,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/1b096565e1764eefb657b07b28904613 is 50, key is test_row_0/C:col10/1731435372109/Put/seqid=0 2024-11-12T18:16:13,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742383_1559 (size=12301) 2024-11-12T18:16:13,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52548 deadline: 1731435433581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52534 deadline: 1731435433581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52520 deadline: 1731435433590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52500 deadline: 1731435433590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 , the old value is region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., hostname=9911683f163c,43757,1731435233300, seqNum=2 because the exception is null or not the one we care about 2024-11-12T18:16:13,718 INFO [Thread-2097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2097. 2024-11-12T18:16:13,718 DEBUG [Thread-2097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:13,719 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:13,719 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,722 INFO [Thread-2099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2099. 2024-11-12T18:16:13,722 DEBUG [Thread-2099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:13,722 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,722 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:13,722 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-11-12T18:16:13,722 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:13,722 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,722 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:13,732 INFO [Thread-2095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2095. 2024-11-12T18:16:13,732 DEBUG [Thread-2095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:13,732 DEBUG [Thread-2095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,732 DEBUG [Thread-2095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:13,736 INFO [Thread-2101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2101. 2024-11-12T18:16:13,736 DEBUG [Thread-2101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:13,736 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,736 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:13,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:13,990 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/1b096565e1764eefb657b07b28904613 2024-11-12T18:16:13,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/c72bfe96585845808437db88945de8a2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c72bfe96585845808437db88945de8a2 2024-11-12T18:16:13,998 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c72bfe96585845808437db88945de8a2, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T18:16:13,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/300a8d0f6e5a4f9a8a81f24fa3ded03b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/300a8d0f6e5a4f9a8a81f24fa3ded03b 2024-11-12T18:16:14,002 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/300a8d0f6e5a4f9a8a81f24fa3ded03b, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T18:16:14,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/1b096565e1764eefb657b07b28904613 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1b096565e1764eefb657b07b28904613 2024-11-12T18:16:14,010 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1b096565e1764eefb657b07b28904613, entries=150, sequenceid=466, filesize=12.0 K 2024-11-12T18:16:14,011 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0f73c86ec05d2a0b97b6becb163767f7 in 1288ms, sequenceid=466, compaction requested=true 2024-11-12T18:16:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-12T18:16:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-12T18:16:14,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-12T18:16:14,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-12T18:16:14,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.0900 sec 2024-11-12T18:16:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T18:16:14,081 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:14,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-12T18:16:14,103 INFO [Thread-2086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2086. 2024-11-12T18:16:14,103 DEBUG [Thread-2086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:14,103 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:14,103 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:14,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:14,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:14,104 INFO [Thread-2090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2090. 2024-11-12T18:16:14,104 DEBUG [Thread-2090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:14,104 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,104 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:14,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/16236fb7797c40ea8a31d4b7248d62d5 is 50, key is test_row_0/A:col10/1731435372928/Put/seqid=0 2024-11-12T18:16:14,112 INFO [Thread-2088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2088. 2024-11-12T18:16:14,112 DEBUG [Thread-2088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:14,112 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,112 DEBUG [Thread-2088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:14,113 INFO [Thread-2084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2084. 2024-11-12T18:16:14,113 DEBUG [Thread-2084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:14,113 DEBUG [Thread-2084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,113 DEBUG [Thread-2084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:14,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742384_1560 (size=12301) 2024-11-12T18:16:14,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/16236fb7797c40ea8a31d4b7248d62d5 2024-11-12T18:16:14,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/2ec746ff26fa4f548c74af3de32059da is 50, key is test_row_0/B:col10/1731435372928/Put/seqid=0 2024-11-12T18:16:14,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742385_1561 (size=12301) 2024-11-12T18:16:14,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/2ec746ff26fa4f548c74af3de32059da 2024-11-12T18:16:14,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/52034cc456034a32a3d5d3b336fdbfdd is 50, key is test_row_0/C:col10/1731435372928/Put/seqid=0 2024-11-12T18:16:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742386_1562 (size=12301) 2024-11-12T18:16:15,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/52034cc456034a32a3d5d3b336fdbfdd 2024-11-12T18:16:15,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/16236fb7797c40ea8a31d4b7248d62d5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/16236fb7797c40ea8a31d4b7248d62d5 2024-11-12T18:16:15,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/16236fb7797c40ea8a31d4b7248d62d5, entries=150, sequenceid=503, filesize=12.0 K 2024-11-12T18:16:15,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/2ec746ff26fa4f548c74af3de32059da as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2ec746ff26fa4f548c74af3de32059da 2024-11-12T18:16:15,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2ec746ff26fa4f548c74af3de32059da, entries=150, sequenceid=503, filesize=12.0 K 2024-11-12T18:16:15,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/52034cc456034a32a3d5d3b336fdbfdd as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/52034cc456034a32a3d5d3b336fdbfdd 2024-11-12T18:16:15,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/52034cc456034a32a3d5d3b336fdbfdd, entries=150, sequenceid=503, filesize=12.0 K 2024-11-12T18:16:15,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=13.42 KB/13740 for 0f73c86ec05d2a0b97b6becb163767f7 in 1262ms, sequenceid=503, compaction requested=true 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:15,365 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f73c86ec05d2a0b97b6becb163767f7:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:15,365 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:15,366 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:15,366 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:15,366 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/B is initiating minor compaction (all files) 2024-11-12T18:16:15,366 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/A is initiating minor compaction (all files) 2024-11-12T18:16:15,366 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/A in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:15,366 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/B in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:15,366 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/3aa5b182cc5f423d9686902a6665a863, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/81f63d27b8da43c1ad7fd98f7f982cbc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c72bfe96585845808437db88945de8a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/16236fb7797c40ea8a31d4b7248d62d5] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=51.4 K 2024-11-12T18:16:15,366 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/74ade584ea814cf2b48ba3ffad3aa32e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/69ec6ba6f7d14ec792f26a95f31b2a17, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/300a8d0f6e5a4f9a8a81f24fa3ded03b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2ec746ff26fa4f548c74af3de32059da] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=49.0 K 2024-11-12T18:16:15,367 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ade584ea814cf2b48ba3ffad3aa32e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731435370095 2024-11-12T18:16:15,367 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa5b182cc5f423d9686902a6665a863, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731435370095 2024-11-12T18:16:15,367 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81f63d27b8da43c1ad7fd98f7f982cbc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731435371381 2024-11-12T18:16:15,367 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ec6ba6f7d14ec792f26a95f31b2a17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731435371400 2024-11-12T18:16:15,367 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72bfe96585845808437db88945de8a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731435372108 2024-11-12T18:16:15,367 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 300a8d0f6e5a4f9a8a81f24fa3ded03b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731435372108 2024-11-12T18:16:15,368 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16236fb7797c40ea8a31d4b7248d62d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731435372928 2024-11-12T18:16:15,368 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec746ff26fa4f548c74af3de32059da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731435372928 2024-11-12T18:16:15,376 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#A#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:15,376 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/9df6b5a21edd47cd865549c474b7269e is 50, key is test_row_0/A:col10/1731435372928/Put/seqid=0 2024-11-12T18:16:15,376 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:15,377 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ca515bf609d34452abbdd9c7e315d34b is 50, key is test_row_0/B:col10/1731435372928/Put/seqid=0 2024-11-12T18:16:15,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742387_1563 (size=13391) 2024-11-12T18:16:15,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742388_1564 (size=13391) 2024-11-12T18:16:15,784 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/ca515bf609d34452abbdd9c7e315d34b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ca515bf609d34452abbdd9c7e315d34b 2024-11-12T18:16:15,784 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/9df6b5a21edd47cd865549c474b7269e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/9df6b5a21edd47cd865549c474b7269e 2024-11-12T18:16:15,789 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/B of 0f73c86ec05d2a0b97b6becb163767f7 into ca515bf609d34452abbdd9c7e315d34b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:15,789 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/A of 0f73c86ec05d2a0b97b6becb163767f7 into 9df6b5a21edd47cd865549c474b7269e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:15,789 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/B, priority=12, startTime=1731435375365; duration=0sec 2024-11-12T18:16:15,789 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/A, priority=12, startTime=1731435375365; duration=0sec 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:B 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:A 2024-11-12T18:16:15,789 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:15,793 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:15,793 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): 0f73c86ec05d2a0b97b6becb163767f7/C is initiating minor compaction (all files) 2024-11-12T18:16:15,793 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f73c86ec05d2a0b97b6becb163767f7/C in TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:15,793 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/71a9c324e4124fd29af9b1a8437a7072, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/aac81db63ac14c0899ab8ada4f59bdce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1b096565e1764eefb657b07b28904613, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/52034cc456034a32a3d5d3b336fdbfdd] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp, totalSize=49.0 K 2024-11-12T18:16:15,793 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a9c324e4124fd29af9b1a8437a7072, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731435370095 2024-11-12T18:16:15,794 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting aac81db63ac14c0899ab8ada4f59bdce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731435371400 2024-11-12T18:16:15,794 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b096565e1764eefb657b07b28904613, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731435372108 2024-11-12T18:16:15,794 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 52034cc456034a32a3d5d3b336fdbfdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731435372928 2024-11-12T18:16:15,803 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f73c86ec05d2a0b97b6becb163767f7#C#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:15,803 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/bdb0f406e7574542bce61fd8350083a5 is 50, key is test_row_0/C:col10/1731435372928/Put/seqid=0 2024-11-12T18:16:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742389_1565 (size=13391) 2024-11-12T18:16:16,183 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-12T18:16:16,183 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:16,183 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:16,183 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:16,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2981 rows 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2970 rows 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2961 rows 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2971 rows 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 987 2024-11-12T18:16:16,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2961 rows 2024-11-12T18:16:16,183 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:16:16,184 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:16:16,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:16,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:16,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:16,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T18:16:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T18:16:16,188 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435376187"}]},"ts":"1731435376187"} 2024-11-12T18:16:16,189 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T18:16:16,189 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T18:16:16,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:16:16,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, UNASSIGN}] 2024-11-12T18:16:16,191 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, UNASSIGN 2024-11-12T18:16:16,192 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0f73c86ec05d2a0b97b6becb163767f7, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:16,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, UNASSIGN because future has completed 2024-11-12T18:16:16,193 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:16:16,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:16:16,211 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/bdb0f406e7574542bce61fd8350083a5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/bdb0f406e7574542bce61fd8350083a5 2024-11-12T18:16:16,215 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f73c86ec05d2a0b97b6becb163767f7/C of 0f73c86ec05d2a0b97b6becb163767f7 into bdb0f406e7574542bce61fd8350083a5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:16,215 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f73c86ec05d2a0b97b6becb163767f7: 2024-11-12T18:16:16,215 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., storeName=0f73c86ec05d2a0b97b6becb163767f7/C, priority=12, startTime=1731435375365; duration=0sec 2024-11-12T18:16:16,215 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:16,215 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f73c86ec05d2a0b97b6becb163767f7:C 2024-11-12T18:16:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T18:16:16,346 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 0f73c86ec05d2a0b97b6becb163767f7, disabling compactions & flushes 2024-11-12T18:16:16,346 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. after waiting 0 ms 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:16,346 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 0f73c86ec05d2a0b97b6becb163767f7 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=A 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=B 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0f73c86ec05d2a0b97b6becb163767f7, store=C 2024-11-12T18:16:16,346 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:16,349 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/670c01d703d64585bb9265ff3aebb36f is 50, key is test_row_0/A:col10/1731435374111/Put/seqid=0 2024-11-12T18:16:16,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742390_1566 (size=12301) 2024-11-12T18:16:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T18:16:16,753 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/670c01d703d64585bb9265ff3aebb36f 2024-11-12T18:16:16,759 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/2faf37774a5245d392b2ad3086211af9 is 50, key is test_row_0/B:col10/1731435374111/Put/seqid=0 2024-11-12T18:16:16,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742391_1567 (size=12301) 2024-11-12T18:16:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T18:16:17,167 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/2faf37774a5245d392b2ad3086211af9 2024-11-12T18:16:17,172 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/f6af85cfa5e44cf2a4465e05564bcaec is 50, key is test_row_0/C:col10/1731435374111/Put/seqid=0 2024-11-12T18:16:17,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742392_1568 (size=12301) 2024-11-12T18:16:17,176 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/f6af85cfa5e44cf2a4465e05564bcaec 2024-11-12T18:16:17,179 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/A/670c01d703d64585bb9265ff3aebb36f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/670c01d703d64585bb9265ff3aebb36f 2024-11-12T18:16:17,182 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/670c01d703d64585bb9265ff3aebb36f, entries=150, sequenceid=512, filesize=12.0 K 2024-11-12T18:16:17,182 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/B/2faf37774a5245d392b2ad3086211af9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2faf37774a5245d392b2ad3086211af9 2024-11-12T18:16:17,185 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2faf37774a5245d392b2ad3086211af9, entries=150, sequenceid=512, filesize=12.0 K 2024-11-12T18:16:17,186 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/.tmp/C/f6af85cfa5e44cf2a4465e05564bcaec as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/f6af85cfa5e44cf2a4465e05564bcaec 2024-11-12T18:16:17,189 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/f6af85cfa5e44cf2a4465e05564bcaec, entries=150, sequenceid=512, filesize=12.0 K 2024-11-12T18:16:17,190 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0f73c86ec05d2a0b97b6becb163767f7 in 843ms, sequenceid=512, compaction requested=false 2024-11-12T18:16:17,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8a770d01d91f425195a330293bf3f453, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/efcb73745241444d9fa10fbd0c988eb2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/da2037b2350c4298915734346d6f8f74, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/7b5453110e7c48fca4912164c780e7c6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/21d74a3e2785407f9a4c599a0f9b5a12, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/ca22f23ce52946e9b3a1adab19e6b113, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/aac4ce1882084cf6895097ded2085fc8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/0e16597dfd5648c3bc6c0849c4907a6f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/abdda80d3fac48e89909801d693b788b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/78f66af8b70147b7b638399241b6ca2c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c3be23f265f34e5bb9d0245d28ff95ff, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5f6990078e7a4b63b023c9743bc68757, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2ea915dd255a4655ba3097fcdbf8f206, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/67f31fa6ac3645929d5d914926b0b39e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/eaad418807de4054af2c048bd5a39f83, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/60bd93530c1e47e890041d8d9e771030, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5c8c8fab752d43c9aa6ad5ffaf567dce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/f34c0cf9bd874c19b4407d795693f75b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/57e6061f3dae4b79b4b908f3f0c9b8ab, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/d8fbf85d47bb4308b7ad24a58d96870f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/e056a96460b24650b4fe691d5c5382f0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/a47307a3b6934f758471c1230e7cb53e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e674194c99a43caba7007ca1e402b64, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/803d3b1f1fba4c29b12f4e9fbc8a6833, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8896ff0b967f4e7cb1e1acf010325068, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/69478f4b13574a3bb8369250981d08f3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e1ac269edb54f12a0b3864c9c612f41, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/673da9f0c853441494dad9642f542713, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/3aa5b182cc5f423d9686902a6665a863, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/81f63d27b8da43c1ad7fd98f7f982cbc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c72bfe96585845808437db88945de8a2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/16236fb7797c40ea8a31d4b7248d62d5] to archive 2024-11-12T18:16:17,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:16:17,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8a770d01d91f425195a330293bf3f453 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8a770d01d91f425195a330293bf3f453 2024-11-12T18:16:17,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/efcb73745241444d9fa10fbd0c988eb2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/efcb73745241444d9fa10fbd0c988eb2 2024-11-12T18:16:17,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/da2037b2350c4298915734346d6f8f74 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/da2037b2350c4298915734346d6f8f74 2024-11-12T18:16:17,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/7b5453110e7c48fca4912164c780e7c6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/7b5453110e7c48fca4912164c780e7c6 2024-11-12T18:16:17,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/21d74a3e2785407f9a4c599a0f9b5a12 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/21d74a3e2785407f9a4c599a0f9b5a12 2024-11-12T18:16:17,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/ca22f23ce52946e9b3a1adab19e6b113 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/ca22f23ce52946e9b3a1adab19e6b113 2024-11-12T18:16:17,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/aac4ce1882084cf6895097ded2085fc8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/aac4ce1882084cf6895097ded2085fc8 2024-11-12T18:16:17,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/0e16597dfd5648c3bc6c0849c4907a6f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/0e16597dfd5648c3bc6c0849c4907a6f 2024-11-12T18:16:17,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/abdda80d3fac48e89909801d693b788b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/abdda80d3fac48e89909801d693b788b 2024-11-12T18:16:17,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/78f66af8b70147b7b638399241b6ca2c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/78f66af8b70147b7b638399241b6ca2c 2024-11-12T18:16:17,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c3be23f265f34e5bb9d0245d28ff95ff to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c3be23f265f34e5bb9d0245d28ff95ff 2024-11-12T18:16:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5f6990078e7a4b63b023c9743bc68757 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5f6990078e7a4b63b023c9743bc68757 2024-11-12T18:16:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2ea915dd255a4655ba3097fcdbf8f206 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2ea915dd255a4655ba3097fcdbf8f206 2024-11-12T18:16:17,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/67f31fa6ac3645929d5d914926b0b39e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/67f31fa6ac3645929d5d914926b0b39e 2024-11-12T18:16:17,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/eaad418807de4054af2c048bd5a39f83 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/eaad418807de4054af2c048bd5a39f83 2024-11-12T18:16:17,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/60bd93530c1e47e890041d8d9e771030 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/60bd93530c1e47e890041d8d9e771030 2024-11-12T18:16:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5c8c8fab752d43c9aa6ad5ffaf567dce to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/5c8c8fab752d43c9aa6ad5ffaf567dce 2024-11-12T18:16:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/f34c0cf9bd874c19b4407d795693f75b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/f34c0cf9bd874c19b4407d795693f75b 2024-11-12T18:16:17,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/57e6061f3dae4b79b4b908f3f0c9b8ab to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/57e6061f3dae4b79b4b908f3f0c9b8ab 2024-11-12T18:16:17,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/d8fbf85d47bb4308b7ad24a58d96870f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/d8fbf85d47bb4308b7ad24a58d96870f 2024-11-12T18:16:17,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/e056a96460b24650b4fe691d5c5382f0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/e056a96460b24650b4fe691d5c5382f0 2024-11-12T18:16:17,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/a47307a3b6934f758471c1230e7cb53e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/a47307a3b6934f758471c1230e7cb53e 2024-11-12T18:16:17,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e674194c99a43caba7007ca1e402b64 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e674194c99a43caba7007ca1e402b64 2024-11-12T18:16:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/803d3b1f1fba4c29b12f4e9fbc8a6833 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/803d3b1f1fba4c29b12f4e9fbc8a6833 2024-11-12T18:16:17,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8896ff0b967f4e7cb1e1acf010325068 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/8896ff0b967f4e7cb1e1acf010325068 2024-11-12T18:16:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/69478f4b13574a3bb8369250981d08f3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/69478f4b13574a3bb8369250981d08f3 2024-11-12T18:16:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e1ac269edb54f12a0b3864c9c612f41 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/2e1ac269edb54f12a0b3864c9c612f41 2024-11-12T18:16:17,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/673da9f0c853441494dad9642f542713 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/673da9f0c853441494dad9642f542713 2024-11-12T18:16:17,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/3aa5b182cc5f423d9686902a6665a863 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/3aa5b182cc5f423d9686902a6665a863 2024-11-12T18:16:17,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/81f63d27b8da43c1ad7fd98f7f982cbc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/81f63d27b8da43c1ad7fd98f7f982cbc 2024-11-12T18:16:17,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c72bfe96585845808437db88945de8a2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/c72bfe96585845808437db88945de8a2 2024-11-12T18:16:17,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/16236fb7797c40ea8a31d4b7248d62d5 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/16236fb7797c40ea8a31d4b7248d62d5 2024-11-12T18:16:17,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/9db373f9590c47f3b934e2b466ec5ea9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a3f2eaf194d496ab58132ce252a4597, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/de96397acb8b4ad799f32fe7ce15a35e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ad419212dca04383a94ca5f362b5d621, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a5ce3b44f1c45e4a909d463fd760818, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/fa2114f3438d4797a3f30d05a565d41b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/e710e8887b0445dda98de5bbdc13d786, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/7db937547ad840f9af7fd2ab6087aeb0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/b9bf79483420417ba89aa198e690865b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/dfc078adc8534a9bab8600cc8f6b3fd0, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cf857fdc4df942f29e3784529d60a0c8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d16326444a8d4750bdf27f82787ac1ec, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/84e266a10b364ad08d2511666b2f650e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1501f1f041ef47b5859c358e425fd45a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/89bcdb76e6d349f3833303dc4c8b8ac1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/eb0fd752217f48dfa589464d1c19264f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4905118144fa4444ab797e055c4ed608, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ec411a29611843a2b03a453b026c5b83, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/c6de3a448e86459eb29242e7db654e8d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cd986ca0cb484a0fab309ad42b1fb0bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4ed26b795fff4cd9bb86b280e8975580, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d0cfe03e83fc417d97f5177cbed45bce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/3e98cbc46b32401aad9f043bdd1cda86, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ffdf0e830e8b42aabcf50a440c1053ad, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1683e19e5e3d477180974566b56c4cc6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/88ac1774713341b49008d72b431cc9a6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/5184325efea74d80859e1567768b2b72, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/74ade584ea814cf2b48ba3ffad3aa32e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/a35c69c3bfa74a65a930a42ab6d6288e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/69ec6ba6f7d14ec792f26a95f31b2a17, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/300a8d0f6e5a4f9a8a81f24fa3ded03b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2ec746ff26fa4f548c74af3de32059da] to archive 2024-11-12T18:16:17,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:16:17,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/9db373f9590c47f3b934e2b466ec5ea9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/9db373f9590c47f3b934e2b466ec5ea9 2024-11-12T18:16:17,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a3f2eaf194d496ab58132ce252a4597 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a3f2eaf194d496ab58132ce252a4597 2024-11-12T18:16:17,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/de96397acb8b4ad799f32fe7ce15a35e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/de96397acb8b4ad799f32fe7ce15a35e 2024-11-12T18:16:17,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ad419212dca04383a94ca5f362b5d621 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ad419212dca04383a94ca5f362b5d621 2024-11-12T18:16:17,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a5ce3b44f1c45e4a909d463fd760818 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4a5ce3b44f1c45e4a909d463fd760818 2024-11-12T18:16:17,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/fa2114f3438d4797a3f30d05a565d41b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/fa2114f3438d4797a3f30d05a565d41b 2024-11-12T18:16:17,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/e710e8887b0445dda98de5bbdc13d786 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/e710e8887b0445dda98de5bbdc13d786 2024-11-12T18:16:17,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/7db937547ad840f9af7fd2ab6087aeb0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/7db937547ad840f9af7fd2ab6087aeb0 2024-11-12T18:16:17,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/b9bf79483420417ba89aa198e690865b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/b9bf79483420417ba89aa198e690865b 2024-11-12T18:16:17,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/dfc078adc8534a9bab8600cc8f6b3fd0 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/dfc078adc8534a9bab8600cc8f6b3fd0 2024-11-12T18:16:17,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cf857fdc4df942f29e3784529d60a0c8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cf857fdc4df942f29e3784529d60a0c8 2024-11-12T18:16:17,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d16326444a8d4750bdf27f82787ac1ec to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d16326444a8d4750bdf27f82787ac1ec 2024-11-12T18:16:17,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/84e266a10b364ad08d2511666b2f650e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/84e266a10b364ad08d2511666b2f650e 2024-11-12T18:16:17,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1501f1f041ef47b5859c358e425fd45a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1501f1f041ef47b5859c358e425fd45a 2024-11-12T18:16:17,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/89bcdb76e6d349f3833303dc4c8b8ac1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/89bcdb76e6d349f3833303dc4c8b8ac1 2024-11-12T18:16:17,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/eb0fd752217f48dfa589464d1c19264f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/eb0fd752217f48dfa589464d1c19264f 2024-11-12T18:16:17,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4905118144fa4444ab797e055c4ed608 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4905118144fa4444ab797e055c4ed608 2024-11-12T18:16:17,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ec411a29611843a2b03a453b026c5b83 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ec411a29611843a2b03a453b026c5b83 2024-11-12T18:16:17,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/c6de3a448e86459eb29242e7db654e8d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/c6de3a448e86459eb29242e7db654e8d 2024-11-12T18:16:17,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cd986ca0cb484a0fab309ad42b1fb0bb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/cd986ca0cb484a0fab309ad42b1fb0bb 2024-11-12T18:16:17,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4ed26b795fff4cd9bb86b280e8975580 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/4ed26b795fff4cd9bb86b280e8975580 2024-11-12T18:16:17,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d0cfe03e83fc417d97f5177cbed45bce to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/d0cfe03e83fc417d97f5177cbed45bce 2024-11-12T18:16:17,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/3e98cbc46b32401aad9f043bdd1cda86 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/3e98cbc46b32401aad9f043bdd1cda86 2024-11-12T18:16:17,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ffdf0e830e8b42aabcf50a440c1053ad to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ffdf0e830e8b42aabcf50a440c1053ad 2024-11-12T18:16:17,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1683e19e5e3d477180974566b56c4cc6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/1683e19e5e3d477180974566b56c4cc6 2024-11-12T18:16:17,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/88ac1774713341b49008d72b431cc9a6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/88ac1774713341b49008d72b431cc9a6 2024-11-12T18:16:17,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/5184325efea74d80859e1567768b2b72 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/5184325efea74d80859e1567768b2b72 2024-11-12T18:16:17,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/74ade584ea814cf2b48ba3ffad3aa32e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/74ade584ea814cf2b48ba3ffad3aa32e 2024-11-12T18:16:17,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/a35c69c3bfa74a65a930a42ab6d6288e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/a35c69c3bfa74a65a930a42ab6d6288e 2024-11-12T18:16:17,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/69ec6ba6f7d14ec792f26a95f31b2a17 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/69ec6ba6f7d14ec792f26a95f31b2a17 2024-11-12T18:16:17,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/300a8d0f6e5a4f9a8a81f24fa3ded03b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/300a8d0f6e5a4f9a8a81f24fa3ded03b 2024-11-12T18:16:17,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2ec746ff26fa4f548c74af3de32059da to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2ec746ff26fa4f548c74af3de32059da 2024-11-12T18:16:17,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6afeb3b33d9d482a836e517d6a83ad1e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/744b0b87fe8e48c29c519370361bd9dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/81dafe16dc334217a3aea37f11796eda, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/562dc07d1fea4c35bd2a4acfe92f20ba, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a5cd9e06cbb04202a8b7068e36dc0f2f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/43a36d0263004d0ebca66514fb2c9f46, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/03505cf7fde640908f34e736feadee87, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/9d43c13782d942689b55503aae9da085, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e586ff8e178d4b409439ae8da7fd0eae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6db13cb748e345509fdeb9dbe9c6c1a7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/625a24af1eea4a5a8ec0cc9d06400152, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a3cb3208946b4b67b28ff82078c2a4df, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/3801d24230cd4143a5ee57d32c69bf3d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ead9a8157e964154bf073d66421d994e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/4da011a66c86403aad319d935f2a2d2b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/89096095cc4944a1ae629e69187308be, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/33b872df7620476aa940763acbe60917, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/04fa882b84094abcb7529792245eee6f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/49d5e3a91fca4befbb27f881f828b459, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/320d5f86f54649a39e1c1283fdeb3b3b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/62f6488a6da244b1ab7a4af27b721f2b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/d437e56cd334498c8b7d9627b32aaf31, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/b81d8df69d41440bacc8698d0b718395, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e8b794a2b11d4d00ad941ef0a6ae1da2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1d4f3f473f534d0a95dca21a066d224a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ab13b0821b16435bb6141ef6b7efdb11, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/7e13015256204876be0515756fdf40d9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/71a9c324e4124fd29af9b1a8437a7072, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/75b2658e430141a5a0084ac2c6069951, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/aac81db63ac14c0899ab8ada4f59bdce, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1b096565e1764eefb657b07b28904613, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/52034cc456034a32a3d5d3b336fdbfdd] to archive 2024-11-12T18:16:17,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:16:17,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6afeb3b33d9d482a836e517d6a83ad1e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6afeb3b33d9d482a836e517d6a83ad1e 2024-11-12T18:16:17,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/744b0b87fe8e48c29c519370361bd9dc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/744b0b87fe8e48c29c519370361bd9dc 2024-11-12T18:16:17,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/81dafe16dc334217a3aea37f11796eda to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/81dafe16dc334217a3aea37f11796eda 2024-11-12T18:16:17,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/562dc07d1fea4c35bd2a4acfe92f20ba to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/562dc07d1fea4c35bd2a4acfe92f20ba 2024-11-12T18:16:17,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a5cd9e06cbb04202a8b7068e36dc0f2f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a5cd9e06cbb04202a8b7068e36dc0f2f 2024-11-12T18:16:17,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/43a36d0263004d0ebca66514fb2c9f46 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/43a36d0263004d0ebca66514fb2c9f46 2024-11-12T18:16:17,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/03505cf7fde640908f34e736feadee87 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/03505cf7fde640908f34e736feadee87 2024-11-12T18:16:17,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/9d43c13782d942689b55503aae9da085 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/9d43c13782d942689b55503aae9da085 2024-11-12T18:16:17,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e586ff8e178d4b409439ae8da7fd0eae to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e586ff8e178d4b409439ae8da7fd0eae 2024-11-12T18:16:17,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6db13cb748e345509fdeb9dbe9c6c1a7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/6db13cb748e345509fdeb9dbe9c6c1a7 2024-11-12T18:16:17,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/625a24af1eea4a5a8ec0cc9d06400152 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/625a24af1eea4a5a8ec0cc9d06400152 2024-11-12T18:16:17,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a3cb3208946b4b67b28ff82078c2a4df to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/a3cb3208946b4b67b28ff82078c2a4df 2024-11-12T18:16:17,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/3801d24230cd4143a5ee57d32c69bf3d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/3801d24230cd4143a5ee57d32c69bf3d 2024-11-12T18:16:17,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ead9a8157e964154bf073d66421d994e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ead9a8157e964154bf073d66421d994e 2024-11-12T18:16:17,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/4da011a66c86403aad319d935f2a2d2b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/4da011a66c86403aad319d935f2a2d2b 2024-11-12T18:16:17,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/89096095cc4944a1ae629e69187308be to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/89096095cc4944a1ae629e69187308be 2024-11-12T18:16:17,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/33b872df7620476aa940763acbe60917 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/33b872df7620476aa940763acbe60917 2024-11-12T18:16:17,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/04fa882b84094abcb7529792245eee6f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/04fa882b84094abcb7529792245eee6f 2024-11-12T18:16:17,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/49d5e3a91fca4befbb27f881f828b459 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/49d5e3a91fca4befbb27f881f828b459 2024-11-12T18:16:17,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/320d5f86f54649a39e1c1283fdeb3b3b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/320d5f86f54649a39e1c1283fdeb3b3b 2024-11-12T18:16:17,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/62f6488a6da244b1ab7a4af27b721f2b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/62f6488a6da244b1ab7a4af27b721f2b 2024-11-12T18:16:17,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/d437e56cd334498c8b7d9627b32aaf31 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/d437e56cd334498c8b7d9627b32aaf31 2024-11-12T18:16:17,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/b81d8df69d41440bacc8698d0b718395 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/b81d8df69d41440bacc8698d0b718395 2024-11-12T18:16:17,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e8b794a2b11d4d00ad941ef0a6ae1da2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/e8b794a2b11d4d00ad941ef0a6ae1da2 2024-11-12T18:16:17,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1d4f3f473f534d0a95dca21a066d224a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1d4f3f473f534d0a95dca21a066d224a 2024-11-12T18:16:17,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ab13b0821b16435bb6141ef6b7efdb11 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/ab13b0821b16435bb6141ef6b7efdb11 2024-11-12T18:16:17,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/7e13015256204876be0515756fdf40d9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/7e13015256204876be0515756fdf40d9 2024-11-12T18:16:17,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/71a9c324e4124fd29af9b1a8437a7072 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/71a9c324e4124fd29af9b1a8437a7072 2024-11-12T18:16:17,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/75b2658e430141a5a0084ac2c6069951 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/75b2658e430141a5a0084ac2c6069951 2024-11-12T18:16:17,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/aac81db63ac14c0899ab8ada4f59bdce to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/aac81db63ac14c0899ab8ada4f59bdce 2024-11-12T18:16:17,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1b096565e1764eefb657b07b28904613 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/1b096565e1764eefb657b07b28904613 2024-11-12T18:16:17,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/52034cc456034a32a3d5d3b336fdbfdd to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/52034cc456034a32a3d5d3b336fdbfdd 2024-11-12T18:16:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T18:16:17,324 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/recovered.edits/515.seqid, newMaxSeqId=515, maxSeqId=1 2024-11-12T18:16:17,324 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. 2024-11-12T18:16:17,324 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 0f73c86ec05d2a0b97b6becb163767f7: Waiting for close lock at 1731435376346Running coprocessor pre-close hooks at 1731435376346Disabling compacts and flushes for region at 1731435376346Disabling writes for close at 1731435376346Obtaining lock to block concurrent updates at 1731435376346Preparing flush snapshotting stores in 0f73c86ec05d2a0b97b6becb163767f7 at 1731435376346Finished memstore snapshotting TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731435376346Flushing stores of TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7. at 1731435376347 (+1 ms)Flushing 0f73c86ec05d2a0b97b6becb163767f7/A: creating writer at 1731435376347Flushing 0f73c86ec05d2a0b97b6becb163767f7/A: appending metadata at 1731435376349 (+2 ms)Flushing 0f73c86ec05d2a0b97b6becb163767f7/A: closing flushed file at 1731435376349Flushing 0f73c86ec05d2a0b97b6becb163767f7/B: creating writer at 1731435376757 (+408 ms)Flushing 0f73c86ec05d2a0b97b6becb163767f7/B: appending metadata at 1731435376759 (+2 ms)Flushing 0f73c86ec05d2a0b97b6becb163767f7/B: closing flushed file at 1731435376759Flushing 0f73c86ec05d2a0b97b6becb163767f7/C: creating writer at 1731435377170 (+411 ms)Flushing 0f73c86ec05d2a0b97b6becb163767f7/C: appending metadata at 1731435377172 (+2 ms)Flushing 0f73c86ec05d2a0b97b6becb163767f7/C: closing flushed file at 1731435377172Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1118e68e: reopening flushed file at 1731435377178 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f8ac76d: reopening flushed file at 1731435377182 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3681814c: reopening flushed file at 1731435377185 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0f73c86ec05d2a0b97b6becb163767f7 in 843ms, sequenceid=512, compaction requested=false at 1731435377190 (+5 ms)Writing region close event to WAL at 1731435377321 (+131 ms)Running coprocessor post-close hooks at 1731435377324 (+3 ms)Closed at 1731435377324 2024-11-12T18:16:17,327 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:17,328 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0f73c86ec05d2a0b97b6becb163767f7, regionState=CLOSED 2024-11-12T18:16:17,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:16:17,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-12T18:16:17,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 0f73c86ec05d2a0b97b6becb163767f7, server=9911683f163c,43757,1731435233300 in 1.1370 sec 2024-11-12T18:16:17,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-12T18:16:17,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f73c86ec05d2a0b97b6becb163767f7, UNASSIGN in 1.1420 sec 2024-11-12T18:16:17,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T18:16:17,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-12T18:16:17,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435377338"}]},"ts":"1731435377338"} 2024-11-12T18:16:17,339 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T18:16:17,340 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T18:16:17,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-12T18:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-12T18:16:18,330 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:18,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T18:16:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,332 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T18:16:18,333 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,335 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:18,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/recovered.edits] 2024-11-12T18:16:18,339 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/670c01d703d64585bb9265ff3aebb36f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/670c01d703d64585bb9265ff3aebb36f 2024-11-12T18:16:18,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/9df6b5a21edd47cd865549c474b7269e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/A/9df6b5a21edd47cd865549c474b7269e 2024-11-12T18:16:18,341 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2faf37774a5245d392b2ad3086211af9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/2faf37774a5245d392b2ad3086211af9 2024-11-12T18:16:18,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ca515bf609d34452abbdd9c7e315d34b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/B/ca515bf609d34452abbdd9c7e315d34b 2024-11-12T18:16:18,344 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/bdb0f406e7574542bce61fd8350083a5 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/bdb0f406e7574542bce61fd8350083a5 2024-11-12T18:16:18,345 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/f6af85cfa5e44cf2a4465e05564bcaec to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/C/f6af85cfa5e44cf2a4465e05564bcaec 2024-11-12T18:16:18,346 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/recovered.edits/515.seqid to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7/recovered.edits/515.seqid 2024-11-12T18:16:18,347 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/0f73c86ec05d2a0b97b6becb163767f7 2024-11-12T18:16:18,347 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T18:16:18,349 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,350 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T18:16:18,352 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T18:16:18,353 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,353 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T18:16:18,353 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731435378353"}]},"ts":"9223372036854775807"} 2024-11-12T18:16:18,355 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T18:16:18,355 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0f73c86ec05d2a0b97b6becb163767f7, NAME => 'TestAcidGuarantees,,1731435351493.0f73c86ec05d2a0b97b6becb163767f7.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T18:16:18,355 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T18:16:18,355 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731435378355"}]},"ts":"9223372036854775807"} 2024-11-12T18:16:18,357 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T18:16:18,357 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-12T18:16:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-12T18:16:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T18:16:18,440 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:18,453 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 251) - Thread LEAK? -, OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=644 (was 755), ProcessCount=11 (was 11), AvailableMemoryMB=476 (was 1642) 2024-11-12T18:16:18,461 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=644, ProcessCount=11, AvailableMemoryMB=476 2024-11-12T18:16:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:16:18,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:16:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:18,465 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T18:16:18,465 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:18,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-12T18:16:18,466 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T18:16:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T18:16:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742393_1569 (size=1061) 2024-11-12T18:16:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T18:16:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T18:16:18,872 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7 2024-11-12T18:16:18,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742394_1570 (size=53) 2024-11-12T18:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T18:16:19,277 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:16:19,277 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ffab90d6ee04bb318f8ea8ffd27a4f4b, disabling compactions & flushes 2024-11-12T18:16:19,277 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,277 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,277 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. after waiting 0 ms 2024-11-12T18:16:19,277 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,277 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,277 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: Waiting for close lock at 1731435379277Disabling compacts and flushes for region at 1731435379277Disabling writes for close at 1731435379277Writing region close event to WAL at 1731435379277Closed at 1731435379277 2024-11-12T18:16:19,278 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T18:16:19,278 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731435379278"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731435379278"}]},"ts":"1731435379278"} 2024-11-12T18:16:19,280 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T18:16:19,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T18:16:19,281 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435379280"}]},"ts":"1731435379280"} 2024-11-12T18:16:19,281 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T18:16:19,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, ASSIGN}] 2024-11-12T18:16:19,283 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, ASSIGN 2024-11-12T18:16:19,283 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, ASSIGN; state=OFFLINE, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=false 2024-11-12T18:16:19,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:19,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, ASSIGN because future has completed 2024-11-12T18:16:19,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:16:19,591 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,591 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:16:19,591 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,591 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:16:19,592 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,592 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,593 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,594 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:16:19,594 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffab90d6ee04bb318f8ea8ffd27a4f4b columnFamilyName A 2024-11-12T18:16:19,594 DEBUG [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:19,594 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(327): Store=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:16:19,594 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,595 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:16:19,595 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffab90d6ee04bb318f8ea8ffd27a4f4b columnFamilyName B 2024-11-12T18:16:19,595 DEBUG [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:19,596 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(327): Store=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:16:19,596 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,596 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:16:19,597 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffab90d6ee04bb318f8ea8ffd27a4f4b columnFamilyName C 2024-11-12T18:16:19,597 DEBUG [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:19,597 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(327): Store=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:16:19,597 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,597 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,598 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,598 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,599 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,599 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,599 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:16:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T18:16:19,601 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,603 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T18:16:19,603 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened ffab90d6ee04bb318f8ea8ffd27a4f4b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67672121, jitterRate=0.00839318335056305}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:16:19,603 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:19,604 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: Running coprocessor pre-open hook at 1731435379592Writing region info on filesystem at 1731435379592Initializing all the Stores at 1731435379592Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435379592Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435379592Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435379592Cleaning up temporary data from old regions at 1731435379599 (+7 ms)Running coprocessor post-open hooks at 1731435379603 (+4 ms)Region opened successfully at 1731435379603 2024-11-12T18:16:19,604 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., pid=144, masterSystemTime=1731435379588 2024-11-12T18:16:19,606 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,606 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:19,606 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=OPEN, openSeqNum=2, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:19,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:16:19,608 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39807 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9911683f163c,43757,1731435233300, table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T18:16:19,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-12T18:16:19,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 in 172 msec 2024-11-12T18:16:19,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-12T18:16:19,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, ASSIGN in 328 msec 2024-11-12T18:16:19,612 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T18:16:19,612 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435379612"}]},"ts":"1731435379612"} 2024-11-12T18:16:19,613 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T18:16:19,616 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T18:16:19,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-12T18:16:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T18:16:20,611 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:20,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:20,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:20,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:20,613 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540c6102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:20,614 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a340620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:20,615 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:20,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:20,618 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T18:16:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9911683f163c,39807,1731435232358 2024-11-12T18:16:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5200f9ee 2024-11-12T18:16:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T18:16:20,620 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T18:16:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T18:16:20,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T18:16:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:20,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742395_1571 (size=1097) 2024-11-12T18:16:21,040 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T18:16:21,040 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T18:16:21,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:16:21,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, REOPEN/MOVE}] 2024-11-12T18:16:21,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, REOPEN/MOVE 2024-11-12T18:16:21,047 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, REOPEN/MOVE because future has completed 2024-11-12T18:16:21,048 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:16:21,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:16:21,201 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,201 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:16:21,201 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing ffab90d6ee04bb318f8ea8ffd27a4f4b, disabling compactions & flushes 2024-11-12T18:16:21,201 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,201 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,201 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. after waiting 0 ms 2024-11-12T18:16:21,201 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,205 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T18:16:21,206 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,206 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: Waiting for close lock at 1731435381201Running coprocessor pre-close hooks at 1731435381201Disabling compacts and flushes for region at 1731435381201Disabling writes for close at 1731435381201Writing region close event to WAL at 1731435381202 (+1 ms)Running coprocessor post-close hooks at 1731435381206 (+4 ms)Closed at 1731435381206 2024-11-12T18:16:21,206 WARN [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: ffab90d6ee04bb318f8ea8ffd27a4f4b to self. 2024-11-12T18:16:21,208 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=CLOSED 2024-11-12T18:16:21,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:16:21,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-12T18:16:21,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 in 163 msec 2024-11-12T18:16:21,214 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, REOPEN/MOVE; state=CLOSED, location=9911683f163c,43757,1731435233300; forceNewPlan=false, retain=true 2024-11-12T18:16:21,242 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T18:16:21,365 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=OPENING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, REOPEN/MOVE because future has completed 2024-11-12T18:16:21,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:16:21,522 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,522 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T18:16:21,522 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,522 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T18:16:21,522 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,522 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,524 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,524 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:16:21,525 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffab90d6ee04bb318f8ea8ffd27a4f4b columnFamilyName A 2024-11-12T18:16:21,526 DEBUG [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:21,526 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(327): Store=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:16:21,526 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,527 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:16:21,527 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffab90d6ee04bb318f8ea8ffd27a4f4b columnFamilyName B 2024-11-12T18:16:21,527 DEBUG [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:21,527 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(327): Store=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:16:21,527 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,528 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T18:16:21,528 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffab90d6ee04bb318f8ea8ffd27a4f4b columnFamilyName C 2024-11-12T18:16:21,528 DEBUG [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:21,528 INFO [StoreOpener-ffab90d6ee04bb318f8ea8ffd27a4f4b-1 {}] regionserver.HStore(327): Store=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T18:16:21,528 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,528 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,529 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,530 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,531 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,531 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,532 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T18:16:21,534 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,535 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened ffab90d6ee04bb318f8ea8ffd27a4f4b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65849343, jitterRate=-0.018768325448036194}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T18:16:21,535 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,536 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: Running coprocessor pre-open hook at 1731435381523Writing region info on filesystem at 1731435381523Initializing all the Stores at 1731435381523Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435381523Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435381524 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731435381524Cleaning up temporary data from old regions at 1731435381531 (+7 ms)Running coprocessor post-open hooks at 1731435381535 (+4 ms)Region opened successfully at 1731435381536 (+1 ms) 2024-11-12T18:16:21,536 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., pid=149, masterSystemTime=1731435381519 2024-11-12T18:16:21,539 DEBUG [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,539 INFO [RS_OPEN_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,540 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=OPEN, openSeqNum=5, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:16:21,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-12T18:16:21,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 in 175 msec 2024-11-12T18:16:21,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-12T18:16:21,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, REOPEN/MOVE in 501 msec 2024-11-12T18:16:21,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-12T18:16:21,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-12T18:16:21,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 930 msec 2024-11-12T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-12T18:16:21,555 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:21,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0f778c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bb8d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,560 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1603b6e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3d0808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e613e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4085c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,568 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de22c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee56ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,571 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7eeea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,573 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537817e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,575 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2f13cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d687233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,578 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d3102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d733cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815f473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52537da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,586 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c134f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777655d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9911683f163c,39807,-1 for getting cluster id 2024-11-12T18:16:21,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T18:16:21,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bcba757-a082-44ff-81f8-05d913183c06' 2024-11-12T18:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T18:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bcba757-a082-44ff-81f8-05d913183c06" 2024-11-12T18:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3bc14d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9911683f163c,39807,-1] 2024-11-12T18:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T18:16:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:21,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T18:16:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d0304c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T18:16:21,592 DEBUG [Thread-2539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,593 DEBUG [Thread-2531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,593 DEBUG [Thread-2537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,594 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,595 DEBUG [Thread-2550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-12T18:16:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,600 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:21,600 DEBUG [Thread-2533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,602 DEBUG [Thread-2546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,603 DEBUG [Thread-2544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:21,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:21,604 DEBUG [Thread-2535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,607 DEBUG [Thread-2535 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,607 DEBUG [Thread-2535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,613 DEBUG [Thread-2535 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,614 DEBUG [Thread-2548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,629 DEBUG [Thread-2542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T18:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:21,631 DEBUG [Thread-2542 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9911683f163c,43757,1731435233300, seqNum=-1] 2024-11-12T18:16:21,651 DEBUG [Thread-2542 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T18:16:21,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T18:16:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5] 2024-11-12T18:16:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435441671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435441675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435441668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435441683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435441685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c2b7968d7e084474a2bf7f2c89050e6c_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435381611/Put/seqid=0 2024-11-12T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:21,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742396_1572 (size=14594) 2024-11-12T18:16:21,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:21,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c2b7968d7e084474a2bf7f2c89050e6c_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c2b7968d7e084474a2bf7f2c89050e6c_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:21,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/c532340d3ebc4e8e9ffb8025456bf3da, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:21,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T18:16:21,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/c532340d3ebc4e8e9ffb8025456bf3da is 175, key is test_row_0/A:col10/1731435381611/Put/seqid=0 2024-11-12T18:16:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:21,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435441790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435441791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742397_1573 (size=39549) 2024-11-12T18:16:21,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/c532340d3ebc4e8e9ffb8025456bf3da 2024-11-12T18:16:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435441801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435441801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435441811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:21,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/f29f7448a50041afa886b88a468c5adb is 50, key is test_row_0/B:col10/1731435381611/Put/seqid=0 2024-11-12T18:16:21,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742398_1574 (size=12001) 2024-11-12T18:16:21,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/f29f7448a50041afa886b88a468c5adb 2024-11-12T18:16:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:21,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T18:16:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:21,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:21,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/77a5f454a9df4cb8af5dfffd8d62e611 is 50, key is test_row_0/C:col10/1731435381611/Put/seqid=0 2024-11-12T18:16:21,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742399_1575 (size=12001) 2024-11-12T18:16:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435442001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435442001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435442010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435442021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435442021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T18:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:22,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T18:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435442317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435442317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435442321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435442330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435442330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/77a5f454a9df4cb8af5dfffd8d62e611 2024-11-12T18:16:22,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/c532340d3ebc4e8e9ffb8025456bf3da as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da 2024-11-12T18:16:22,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da, entries=200, sequenceid=17, filesize=38.6 K 2024-11-12T18:16:22,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/f29f7448a50041afa886b88a468c5adb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f29f7448a50041afa886b88a468c5adb 2024-11-12T18:16:22,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T18:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:22,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f29f7448a50041afa886b88a468c5adb, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T18:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/77a5f454a9df4cb8af5dfffd8d62e611 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/77a5f454a9df4cb8af5dfffd8d62e611 2024-11-12T18:16:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:22,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/77a5f454a9df4cb8af5dfffd8d62e611, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T18:16:22,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 770ms, sequenceid=17, compaction requested=false 2024-11-12T18:16:22,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:22,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:22,544 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112145b0e10519a4f00b6d7b7c17e966843_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435381671/Put/seqid=0 2024-11-12T18:16:22,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742400_1576 (size=12154) 2024-11-12T18:16:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:22,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435442841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,850 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T18:16:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435442844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435442847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435442848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435442851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,963 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112145b0e10519a4f00b6d7b7c17e966843_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112145b0e10519a4f00b6d7b7c17e966843_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435442960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435442960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435442960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435442961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/04199cf9f3864f73ab89916e71592379, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/04199cf9f3864f73ab89916e71592379 is 175, key is test_row_0/A:col10/1731435381671/Put/seqid=0 2024-11-12T18:16:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742401_1577 (size=30955) 2024-11-12T18:16:22,971 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/04199cf9f3864f73ab89916e71592379 2024-11-12T18:16:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/f01f612344314f5896dfceeff4197f10 is 50, key is test_row_0/B:col10/1731435381671/Put/seqid=0 2024-11-12T18:16:23,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742402_1578 (size=12001) 2024-11-12T18:16:23,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435443170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435443171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435443171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435443171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,413 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/f01f612344314f5896dfceeff4197f10 2024-11-12T18:16:23,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e6347e40fc19490fac5b87344228ea45 is 50, key is test_row_0/C:col10/1731435381671/Put/seqid=0 2024-11-12T18:16:23,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742403_1579 (size=12001) 2024-11-12T18:16:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435443480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435443480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435443480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435443480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:23,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e6347e40fc19490fac5b87344228ea45 2024-11-12T18:16:23,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/04199cf9f3864f73ab89916e71592379 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379 2024-11-12T18:16:23,845 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379, entries=150, sequenceid=37, filesize=30.2 K 2024-11-12T18:16:23,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/f01f612344314f5896dfceeff4197f10 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f01f612344314f5896dfceeff4197f10 2024-11-12T18:16:23,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f01f612344314f5896dfceeff4197f10, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T18:16:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e6347e40fc19490fac5b87344228ea45 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e6347e40fc19490fac5b87344228ea45 2024-11-12T18:16:23,866 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e6347e40fc19490fac5b87344228ea45, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T18:16:23,867 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1323ms, sequenceid=37, compaction requested=false 2024-11-12T18:16:23,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:23,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:23,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-12T18:16:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-12T18:16:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:23,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T18:16:23,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:16:23,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-12T18:16:23,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:23,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:23,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:23,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:23,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:23,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:23,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2780 sec 2024-11-12T18:16:23,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125376665f1986403e8ca792dfc338b327_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742404_1580 (size=14594) 2024-11-12T18:16:23,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:23,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125376665f1986403e8ca792dfc338b327_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125376665f1986403e8ca792dfc338b327_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:23,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:23,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b is 175, key is test_row_0/A:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:23,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742405_1581 (size=39549) 2024-11-12T18:16:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435443964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435443990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435443990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435443991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435443991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435444081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435444290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:24,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b 2024-11-12T18:16:24,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/06abb00aa358425f8bf47d5b3e38cd53 is 50, key is test_row_0/B:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:24,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742406_1582 (size=12001) 2024-11-12T18:16:24,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/06abb00aa358425f8bf47d5b3e38cd53 2024-11-12T18:16:24,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/69836a772ced4302b122e646a673b7e1 is 50, key is test_row_0/C:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:24,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742407_1583 (size=12001) 2024-11-12T18:16:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435444601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:24,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/69836a772ced4302b122e646a673b7e1 2024-11-12T18:16:24,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b 2024-11-12T18:16:24,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b, entries=200, sequenceid=51, filesize=38.6 K 2024-11-12T18:16:24,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/06abb00aa358425f8bf47d5b3e38cd53 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/06abb00aa358425f8bf47d5b3e38cd53 2024-11-12T18:16:24,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/06abb00aa358425f8bf47d5b3e38cd53, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T18:16:24,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/69836a772ced4302b122e646a673b7e1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69836a772ced4302b122e646a673b7e1 2024-11-12T18:16:24,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69836a772ced4302b122e646a673b7e1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T18:16:24,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 918ms, sequenceid=51, compaction requested=true 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:24,789 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:24,789 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:24,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:24,790 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:24,790 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:24,790 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:24,790 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=107.5 K 2024-11-12T18:16:24,790 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:24,790 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b] 2024-11-12T18:16:24,791 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:24,791 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:24,791 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:24,791 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f29f7448a50041afa886b88a468c5adb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f01f612344314f5896dfceeff4197f10, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/06abb00aa358425f8bf47d5b3e38cd53] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=35.2 K 2024-11-12T18:16:24,797 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c532340d3ebc4e8e9ffb8025456bf3da, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731435381611 2024-11-12T18:16:24,797 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f29f7448a50041afa886b88a468c5adb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731435381611 2024-11-12T18:16:24,801 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04199cf9f3864f73ab89916e71592379, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731435381662 2024-11-12T18:16:24,801 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f01f612344314f5896dfceeff4197f10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731435381662 2024-11-12T18:16:24,801 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4b1a73ca7eb4f26a00d2c7bf7ee451b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435382842 2024-11-12T18:16:24,801 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 06abb00aa358425f8bf47d5b3e38cd53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435382845 2024-11-12T18:16:24,813 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:24,815 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:24,815 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/30460af477664e95a826d61960f17ae4 is 50, key is test_row_0/B:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:24,821 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112b1542960dfa142a489c54a2b89118f7d_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:24,823 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112b1542960dfa142a489c54a2b89118f7d_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:24,824 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b1542960dfa142a489c54a2b89118f7d_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:24,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742408_1584 (size=12104) 2024-11-12T18:16:24,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742409_1585 (size=4469) 2024-11-12T18:16:24,850 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#501 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:24,851 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/84590914d15d40ffa62641fb319a7500 is 175, key is test_row_0/A:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742410_1586 (size=31058) 2024-11-12T18:16:24,923 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/84590914d15d40ffa62641fb319a7500 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84590914d15d40ffa62641fb319a7500 2024-11-12T18:16:24,938 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into 84590914d15d40ffa62641fb319a7500(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:24,938 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:24,938 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=13, startTime=1731435384789; duration=0sec 2024-11-12T18:16:24,938 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:24,938 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:24,938 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:24,939 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:24,939 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:24,939 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:24,939 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/77a5f454a9df4cb8af5dfffd8d62e611, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e6347e40fc19490fac5b87344228ea45, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69836a772ced4302b122e646a673b7e1] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=35.2 K 2024-11-12T18:16:24,940 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a5f454a9df4cb8af5dfffd8d62e611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731435381611 2024-11-12T18:16:24,940 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6347e40fc19490fac5b87344228ea45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731435381662 2024-11-12T18:16:24,940 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69836a772ced4302b122e646a673b7e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435382845 2024-11-12T18:16:24,954 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:24,954 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/cc3c148735994a92b77b863634d9d3df is 50, key is test_row_0/C:col10/1731435382847/Put/seqid=0 2024-11-12T18:16:24,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742411_1587 (size=12104) 2024-11-12T18:16:25,006 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/cc3c148735994a92b77b863634d9d3df as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/cc3c148735994a92b77b863634d9d3df 2024-11-12T18:16:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:25,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T18:16:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:25,023 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into cc3c148735994a92b77b863634d9d3df(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:25,023 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:25,023 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=13, startTime=1731435384789; duration=0sec 2024-11-12T18:16:25,024 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:25,024 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:25,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e1373a3e0d15407d97944ef0dd7126c9_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435383952/Put/seqid=0 2024-11-12T18:16:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435445062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742412_1588 (size=12154) 2024-11-12T18:16:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435445065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435445066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435445070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:25,087 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e1373a3e0d15407d97944ef0dd7126c9_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e1373a3e0d15407d97944ef0dd7126c9_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:25,088 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4947390725be40dc94230499a8f5718d, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:25,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4947390725be40dc94230499a8f5718d is 175, key is test_row_0/A:col10/1731435383952/Put/seqid=0 2024-11-12T18:16:25,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742413_1589 (size=30955) 2024-11-12T18:16:25,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435445111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435445181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435445181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435445185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435445185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,258 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/30460af477664e95a826d61960f17ae4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/30460af477664e95a826d61960f17ae4 2024-11-12T18:16:25,265 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into 30460af477664e95a826d61960f17ae4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:25,265 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:25,265 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=13, startTime=1731435384789; duration=0sec 2024-11-12T18:16:25,265 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:25,265 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435445390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435445391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435445400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435445401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=71, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4947390725be40dc94230499a8f5718d 2024-11-12T18:16:25,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/17895edafd2749079a6d6af0e5db80d6 is 50, key is test_row_0/B:col10/1731435383952/Put/seqid=0 2024-11-12T18:16:25,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742414_1590 (size=12001) 2024-11-12T18:16:25,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/17895edafd2749079a6d6af0e5db80d6 2024-11-12T18:16:25,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/ff1728ab47ec4baf9f81886505a57a18 is 50, key is test_row_0/C:col10/1731435383952/Put/seqid=0 2024-11-12T18:16:25,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742415_1591 (size=12001) 2024-11-12T18:16:25,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/ff1728ab47ec4baf9f81886505a57a18 2024-11-12T18:16:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4947390725be40dc94230499a8f5718d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d 2024-11-12T18:16:25,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d, entries=150, sequenceid=71, filesize=30.2 K 2024-11-12T18:16:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/17895edafd2749079a6d6af0e5db80d6 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/17895edafd2749079a6d6af0e5db80d6 2024-11-12T18:16:25,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/17895edafd2749079a6d6af0e5db80d6, entries=150, sequenceid=71, filesize=11.7 K 2024-11-12T18:16:25,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/ff1728ab47ec4baf9f81886505a57a18 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ff1728ab47ec4baf9f81886505a57a18 2024-11-12T18:16:25,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ff1728ab47ec4baf9f81886505a57a18, entries=150, sequenceid=71, filesize=11.7 K 2024-11-12T18:16:25,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 653ms, sequenceid=71, compaction requested=false 2024-11-12T18:16:25,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:25,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T18:16:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:25,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:25,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112624f8765d8a74f0aac081341682e3f94_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:25,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742416_1592 (size=14594) 2024-11-12T18:16:25,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435445737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435445746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435445749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435445750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T18:16:25,781 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:25,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-12T18:16:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:25,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:25,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:25,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:25,795 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112624f8765d8a74f0aac081341682e3f94_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112624f8765d8a74f0aac081341682e3f94_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:25,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/84bb2da5c38c43228d4c0c93c118c03c, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:25,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/84bb2da5c38c43228d4c0c93c118c03c is 175, key is test_row_0/A:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:25,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742417_1593 (size=39549) 2024-11-12T18:16:25,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/84bb2da5c38c43228d4c0c93c118c03c 2024-11-12T18:16:25,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/4e3a80dd1f8b49ff8b1c645227265a48 is 50, key is test_row_0/B:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435445862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435445862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435445862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435445862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:25,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742418_1594 (size=12001) 2024-11-12T18:16:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:25,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:25,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:25,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435446071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435446071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435446071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435446073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:26,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435446133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,141 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/4e3a80dd1f8b49ff8b1c645227265a48 2024-11-12T18:16:26,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/f643dd18b5ae442abd115d7bea1caebf is 50, key is test_row_0/C:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:26,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742419_1595 (size=12001) 2024-11-12T18:16:26,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/f643dd18b5ae442abd115d7bea1caebf 2024-11-12T18:16:26,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/84bb2da5c38c43228d4c0c93c118c03c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c 2024-11-12T18:16:26,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c, entries=200, sequenceid=88, filesize=38.6 K 2024-11-12T18:16:26,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/4e3a80dd1f8b49ff8b1c645227265a48 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/4e3a80dd1f8b49ff8b1c645227265a48 2024-11-12T18:16:26,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/4e3a80dd1f8b49ff8b1c645227265a48, entries=150, sequenceid=88, filesize=11.7 K 2024-11-12T18:16:26,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/f643dd18b5ae442abd115d7bea1caebf as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f643dd18b5ae442abd115d7bea1caebf 2024-11-12T18:16:26,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f643dd18b5ae442abd115d7bea1caebf, entries=150, sequenceid=88, filesize=11.7 K 2024-11-12T18:16:26,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 678ms, sequenceid=88, compaction requested=true 2024-11-12T18:16:26,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:26,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:26,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:26,384 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:26,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:26,384 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:26,385 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:26,385 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:26,385 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:26,385 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:26,385 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,385 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,385 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/30460af477664e95a826d61960f17ae4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/17895edafd2749079a6d6af0e5db80d6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/4e3a80dd1f8b49ff8b1c645227265a48] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=35.3 K 2024-11-12T18:16:26,385 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84590914d15d40ffa62641fb319a7500, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=99.2 K 2024-11-12T18:16:26,385 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,385 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84590914d15d40ffa62641fb319a7500, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c] 2024-11-12T18:16:26,385 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 30460af477664e95a826d61960f17ae4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435382845 2024-11-12T18:16:26,386 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84590914d15d40ffa62641fb319a7500, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435382845 2024-11-12T18:16:26,386 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 17895edafd2749079a6d6af0e5db80d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731435383945 2024-11-12T18:16:26,386 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4947390725be40dc94230499a8f5718d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731435383945 2024-11-12T18:16:26,386 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e3a80dd1f8b49ff8b1c645227265a48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731435385023 2024-11-12T18:16:26,386 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84bb2da5c38c43228d4c0c93c118c03c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731435385023 2024-11-12T18:16:26,391 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:26,392 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:26,393 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/d8c164be06b04da3837c9cd9c01e0d22 is 50, key is test_row_0/B:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:26,394 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d2228c1ca1b241b6a49a7c91dc5f02e3_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:26,396 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d2228c1ca1b241b6a49a7c91dc5f02e3_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:26,396 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2228c1ca1b241b6a49a7c91dc5f02e3_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:26,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:16:26,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:26,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:26,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:26,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:26,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:26,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:26,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742420_1596 (size=12207) 2024-11-12T18:16:26,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435446414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435446415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435446417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d85c2fc6868f4fb39cf3538ea08b5e13_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435386396/Put/seqid=0 2024-11-12T18:16:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435446419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742421_1597 (size=4469) 2024-11-12T18:16:26,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742422_1598 (size=14594) 2024-11-12T18:16:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435446530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435446530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435446531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435446531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435446740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435446741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435446741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435446741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:26,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,833 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:26,834 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/fbf15db17845446497726852af3afdd9 is 175, key is test_row_0/A:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:26,837 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/d8c164be06b04da3837c9cd9c01e0d22 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/d8c164be06b04da3837c9cd9c01e0d22 2024-11-12T18:16:26,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:26,845 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into d8c164be06b04da3837c9cd9c01e0d22(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:26,845 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:26,845 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=13, startTime=1731435386384; duration=0sec 2024-11-12T18:16:26,845 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:26,845 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:26,845 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:26,856 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d85c2fc6868f4fb39cf3538ea08b5e13_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d85c2fc6868f4fb39cf3538ea08b5e13_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:26,860 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:26,861 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:26,861 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,861 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/cc3c148735994a92b77b863634d9d3df, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ff1728ab47ec4baf9f81886505a57a18, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f643dd18b5ae442abd115d7bea1caebf] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=35.3 K 2024-11-12T18:16:26,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/e908a4c6835f4f9ba59d84673a3fb8da, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:26,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/e908a4c6835f4f9ba59d84673a3fb8da is 175, key is test_row_0/A:col10/1731435386396/Put/seqid=0 2024-11-12T18:16:26,863 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting cc3c148735994a92b77b863634d9d3df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731435382845 2024-11-12T18:16:26,863 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1728ab47ec4baf9f81886505a57a18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1731435383945 2024-11-12T18:16:26,864 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f643dd18b5ae442abd115d7bea1caebf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731435385023 2024-11-12T18:16:26,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742423_1599 (size=31161) 2024-11-12T18:16:26,902 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:26,903 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/69c10980877b4c189ea9b7ad1007aaef is 50, key is test_row_0/C:col10/1731435385702/Put/seqid=0 2024-11-12T18:16:26,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742424_1600 (size=39549) 2024-11-12T18:16:26,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/e908a4c6835f4f9ba59d84673a3fb8da 2024-11-12T18:16:26,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:26,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:26,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/ddf8297ce9c34095a62325e81414dae4 is 50, key is test_row_0/B:col10/1731435386396/Put/seqid=0 2024-11-12T18:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:26,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742425_1601 (size=12207) 2024-11-12T18:16:26,950 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/69c10980877b4c189ea9b7ad1007aaef as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69c10980877b4c189ea9b7ad1007aaef 2024-11-12T18:16:26,958 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into 69c10980877b4c189ea9b7ad1007aaef(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:26,958 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:26,958 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=13, startTime=1731435386385; duration=0sec 2024-11-12T18:16:26,958 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:26,958 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:26,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742426_1602 (size=12001) 2024-11-12T18:16:26,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/ddf8297ce9c34095a62325e81414dae4 2024-11-12T18:16:27,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/36b589f16928490ca57b08b7abbfd1bb is 50, key is test_row_0/C:col10/1731435386396/Put/seqid=0 2024-11-12T18:16:27,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742427_1603 (size=12001) 2024-11-12T18:16:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435447051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435447051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435447051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435447051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,276 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/fbf15db17845446497726852af3afdd9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/fbf15db17845446497726852af3afdd9 2024-11-12T18:16:27,282 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into fbf15db17845446497726852af3afdd9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:27,282 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:27,282 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=13, startTime=1731435386384; duration=0sec 2024-11-12T18:16:27,282 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:27,282 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:27,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:27,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/36b589f16928490ca57b08b7abbfd1bb 2024-11-12T18:16:27,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/e908a4c6835f4f9ba59d84673a3fb8da as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da 2024-11-12T18:16:27,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da, entries=200, sequenceid=108, filesize=38.6 K 2024-11-12T18:16:27,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/ddf8297ce9c34095a62325e81414dae4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ddf8297ce9c34095a62325e81414dae4 2024-11-12T18:16:27,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ddf8297ce9c34095a62325e81414dae4, entries=150, sequenceid=108, filesize=11.7 K 2024-11-12T18:16:27,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/36b589f16928490ca57b08b7abbfd1bb as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36b589f16928490ca57b08b7abbfd1bb 2024-11-12T18:16:27,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36b589f16928490ca57b08b7abbfd1bb, entries=150, sequenceid=108, filesize=11.7 K 2024-11-12T18:16:27,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1073ms, sequenceid=108, compaction requested=false 2024-11-12T18:16:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:27,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:27,533 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126307607727244766858f2bfccec1af2e_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435386415/Put/seqid=0 2024-11-12T18:16:27,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742428_1604 (size=12154) 2024-11-12T18:16:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:27,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435447587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731435387577, max timestamp=1731435387592Num uniques 859; 2024-11-12T18:16:27,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:16:27,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731435387577, max timestamp=1731435387592Num uniques 859; 2024-11-12T18:16:27,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:16:27,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731435387577, max timestamp=1731435387592Num uniques 859; 2024-11-12T18:16:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T18:16:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435447623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435447626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435447627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435447700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435447730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435447744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435447745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435447910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435447950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435447950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435447950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:27,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:27,964 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126307607727244766858f2bfccec1af2e_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126307607727244766858f2bfccec1af2e_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f14fe48a6030413fa4af51a2f4459f9f, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:27,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f14fe48a6030413fa4af51a2f4459f9f is 175, key is test_row_0/A:col10/1731435386415/Put/seqid=0 2024-11-12T18:16:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742429_1605 (size=30955) 2024-11-12T18:16:28,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f14fe48a6030413fa4af51a2f4459f9f 2024-11-12T18:16:28,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e3783c94c56049bdb8f593d455299390 is 50, key is test_row_0/B:col10/1731435386415/Put/seqid=0 2024-11-12T18:16:28,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742430_1606 (size=12001) 2024-11-12T18:16:28,076 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e3783c94c56049bdb8f593d455299390 2024-11-12T18:16:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/828b6eedceba437a8ca963da08b090a7 is 50, key is test_row_0/C:col10/1731435386415/Put/seqid=0 2024-11-12T18:16:28,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742431_1607 (size=12001) 2024-11-12T18:16:28,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435448160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,163 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435448221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435448260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435448260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435448260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,541 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/828b6eedceba437a8ca963da08b090a7 2024-11-12T18:16:28,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/f14fe48a6030413fa4af51a2f4459f9f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f 2024-11-12T18:16:28,551 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f, entries=150, sequenceid=124, filesize=30.2 K 2024-11-12T18:16:28,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e3783c94c56049bdb8f593d455299390 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e3783c94c56049bdb8f593d455299390 2024-11-12T18:16:28,558 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e3783c94c56049bdb8f593d455299390, entries=150, sequenceid=124, filesize=11.7 K 2024-11-12T18:16:28,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/828b6eedceba437a8ca963da08b090a7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/828b6eedceba437a8ca963da08b090a7 2024-11-12T18:16:28,568 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/828b6eedceba437a8ca963da08b090a7, entries=150, sequenceid=124, filesize=11.7 K 2024-11-12T18:16:28,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1035ms, sequenceid=124, compaction requested=true 2024-11-12T18:16:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-12T18:16:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-12T18:16:28,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-12T18:16:28,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7830 sec 2024-11-12T18:16:28,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.7900 sec 2024-11-12T18:16:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:28,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-12T18:16:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:28,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129aada533b1b4457ab4805de551f50ec7_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:28,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742432_1608 (size=14794) 2024-11-12T18:16:28,765 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129aada533b1b4457ab4805de551f50ec7_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129aada533b1b4457ab4805de551f50ec7_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:28,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/3849ffd3cdae429ab14f26f361a71346, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:28,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/3849ffd3cdae429ab14f26f361a71346 is 175, key is test_row_0/A:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742433_1609 (size=39749) 2024-11-12T18:16:28,778 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/3849ffd3cdae429ab14f26f361a71346 2024-11-12T18:16:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435448776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435448778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/2c8ae745de8b4b87b3933cf575b8ea2c is 50, key is test_row_0/B:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435448779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435448781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742434_1610 (size=12151) 2024-11-12T18:16:28,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/2c8ae745de8b4b87b3933cf575b8ea2c 2024-11-12T18:16:28,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/b27e3316fce948529d3251b38c8d85ae is 50, key is test_row_0/C:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:28,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742435_1611 (size=12151) 2024-11-12T18:16:28,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435448890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435448890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435448890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435448891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435449100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435449100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435449100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435449101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/b27e3316fce948529d3251b38c8d85ae 2024-11-12T18:16:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/3849ffd3cdae429ab14f26f361a71346 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346 2024-11-12T18:16:29,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346, entries=200, sequenceid=158, filesize=38.8 K 2024-11-12T18:16:29,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/2c8ae745de8b4b87b3933cf575b8ea2c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2c8ae745de8b4b87b3933cf575b8ea2c 2024-11-12T18:16:29,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2c8ae745de8b4b87b3933cf575b8ea2c, entries=150, sequenceid=158, filesize=11.9 K 2024-11-12T18:16:29,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/b27e3316fce948529d3251b38c8d85ae as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b27e3316fce948529d3251b38c8d85ae 2024-11-12T18:16:29,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b27e3316fce948529d3251b38c8d85ae, entries=150, sequenceid=158, filesize=11.9 K 2024-11-12T18:16:29,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 499ms, sequenceid=158, compaction requested=true 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:29,235 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:29,235 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:29,237 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:29,237 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:29,237 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:29,237 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/d8c164be06b04da3837c9cd9c01e0d22, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ddf8297ce9c34095a62325e81414dae4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e3783c94c56049bdb8f593d455299390, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2c8ae745de8b4b87b3933cf575b8ea2c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=47.2 K 2024-11-12T18:16:29,237 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c164be06b04da3837c9cd9c01e0d22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731435385023 2024-11-12T18:16:29,237 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:29,238 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:29,238 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:29,238 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/fbf15db17845446497726852af3afdd9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=138.1 K 2024-11-12T18:16:29,238 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf8297ce9c34095a62325e81414dae4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731435385723 2024-11-12T18:16:29,238 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:29,238 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/fbf15db17845446497726852af3afdd9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346] 2024-11-12T18:16:29,238 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e3783c94c56049bdb8f593d455299390, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731435386412 2024-11-12T18:16:29,238 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf15db17845446497726852af3afdd9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731435385023 2024-11-12T18:16:29,239 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8ae745de8b4b87b3933cf575b8ea2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435387624 2024-11-12T18:16:29,239 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e908a4c6835f4f9ba59d84673a3fb8da, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731435385723 2024-11-12T18:16:29,239 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting f14fe48a6030413fa4af51a2f4459f9f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731435386412 2024-11-12T18:16:29,239 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3849ffd3cdae429ab14f26f361a71346, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435387611 2024-11-12T18:16:29,250 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:29,251 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/739f60ac0f0c4308b256f9758e058925 is 50, key is test_row_0/B:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:29,253 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:29,261 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112f54d7adeffa94aacb35545b058453221_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:29,263 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112f54d7adeffa94aacb35545b058453221_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:29,263 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f54d7adeffa94aacb35545b058453221_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:29,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742436_1612 (size=12493) 2024-11-12T18:16:29,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742437_1613 (size=4469) 2024-11-12T18:16:29,294 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#522 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:29,294 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/398be3ed29854bed9168550f38149f7b is 175, key is test_row_0/A:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:29,300 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/739f60ac0f0c4308b256f9758e058925 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/739f60ac0f0c4308b256f9758e058925 2024-11-12T18:16:29,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742438_1614 (size=31447) 2024-11-12T18:16:29,305 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into 739f60ac0f0c4308b256f9758e058925(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:29,305 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:29,305 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=12, startTime=1731435389235; duration=0sec 2024-11-12T18:16:29,305 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:29,305 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:29,305 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:29,307 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:29,307 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:29,307 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:29,307 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69c10980877b4c189ea9b7ad1007aaef, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36b589f16928490ca57b08b7abbfd1bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/828b6eedceba437a8ca963da08b090a7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b27e3316fce948529d3251b38c8d85ae] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=47.2 K 2024-11-12T18:16:29,307 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c10980877b4c189ea9b7ad1007aaef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731435385023 2024-11-12T18:16:29,308 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b589f16928490ca57b08b7abbfd1bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731435385723 2024-11-12T18:16:29,308 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 828b6eedceba437a8ca963da08b090a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731435386412 2024-11-12T18:16:29,308 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b27e3316fce948529d3251b38c8d85ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435387624 2024-11-12T18:16:29,320 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:29,321 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe is 50, key is test_row_0/C:col10/1731435388734/Put/seqid=0 2024-11-12T18:16:29,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742439_1615 (size=12493) 2024-11-12T18:16:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:29,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:16:29,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:29,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:29,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:29,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:29,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:29,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:29,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122695cc3d32144a9a80dd89f917452f13_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435389412/Put/seqid=0 2024-11-12T18:16:29,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742440_1616 (size=14794) 2024-11-12T18:16:29,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435449440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435449440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435449442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435449442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435449550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435449550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435449550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435449550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,710 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/398be3ed29854bed9168550f38149f7b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/398be3ed29854bed9168550f38149f7b 2024-11-12T18:16:29,714 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into 398be3ed29854bed9168550f38149f7b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:29,714 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:29,714 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=12, startTime=1731435389235; duration=0sec 2024-11-12T18:16:29,714 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:29,714 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:29,733 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe 2024-11-12T18:16:29,746 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into e8c03468bb1e4dcbb0c3e7f9f0d22fbe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:29,746 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:29,746 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=12, startTime=1731435389235; duration=0sec 2024-11-12T18:16:29,746 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:29,746 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:29,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435449765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435449765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435449765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435449766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:29,830 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:29,836 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122695cc3d32144a9a80dd89f917452f13_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122695cc3d32144a9a80dd89f917452f13_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:29,837 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/91a5efc8b3b94585a7d9088f35e240e3, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:29,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/91a5efc8b3b94585a7d9088f35e240e3 is 175, key is test_row_0/A:col10/1731435389412/Put/seqid=0 2024-11-12T18:16:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742441_1617 (size=39749) 2024-11-12T18:16:29,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/91a5efc8b3b94585a7d9088f35e240e3 2024-11-12T18:16:29,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e4b8cd9ae6c54dbea197eaf0e91a26b9 is 50, key is test_row_0/B:col10/1731435389412/Put/seqid=0 2024-11-12T18:16:29,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742442_1618 (size=12151) 2024-11-12T18:16:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T18:16:29,951 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:29,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-12T18:16:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T18:16:29,954 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:29,955 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:29,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T18:16:30,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435450081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435450081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435450081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435450081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T18:16:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T18:16:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T18:16:30,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e4b8cd9ae6c54dbea197eaf0e91a26b9 2024-11-12T18:16:30,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/be2051941af64cf88fef4df3d2592063 is 50, key is test_row_0/C:col10/1731435389412/Put/seqid=0 2024-11-12T18:16:30,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742443_1619 (size=12151) 2024-11-12T18:16:30,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T18:16:30,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:30,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T18:16:30,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T18:16:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435450590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435450590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435450590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435450590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:30,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/be2051941af64cf88fef4df3d2592063 2024-11-12T18:16:30,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T18:16:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:30,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/91a5efc8b3b94585a7d9088f35e240e3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3 2024-11-12T18:16:30,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3, entries=200, sequenceid=174, filesize=38.8 K 2024-11-12T18:16:30,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e4b8cd9ae6c54dbea197eaf0e91a26b9 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e4b8cd9ae6c54dbea197eaf0e91a26b9 2024-11-12T18:16:30,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e4b8cd9ae6c54dbea197eaf0e91a26b9, entries=150, sequenceid=174, filesize=11.9 K 2024-11-12T18:16:30,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/be2051941af64cf88fef4df3d2592063 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/be2051941af64cf88fef4df3d2592063 2024-11-12T18:16:30,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/be2051941af64cf88fef4df3d2592063, entries=150, sequenceid=174, filesize=11.9 K 2024-11-12T18:16:30,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1346ms, sequenceid=174, compaction requested=false 2024-11-12T18:16:30,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:30,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T18:16:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:30,902 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T18:16:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:30,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112428a1d897757488387457f570e5aef66_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435389440/Put/seqid=0 2024-11-12T18:16:30,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742444_1620 (size=12304) 2024-11-12T18:16:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:30,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112428a1d897757488387457f570e5aef66_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112428a1d897757488387457f570e5aef66_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:30,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/2df9ef720deb43aba64946812e6f9ef4, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/2df9ef720deb43aba64946812e6f9ef4 is 175, key is test_row_0/A:col10/1731435389440/Put/seqid=0 2024-11-12T18:16:30,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742445_1621 (size=31105) 2024-11-12T18:16:30,958 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/2df9ef720deb43aba64946812e6f9ef4 2024-11-12T18:16:30,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/97f4e19619cc40c2a2378049392f120d is 50, key is test_row_0/B:col10/1731435389440/Put/seqid=0 2024-11-12T18:16:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742446_1622 (size=12151) 2024-11-12T18:16:30,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/97f4e19619cc40c2a2378049392f120d 2024-11-12T18:16:30,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/36d2c9db34a14a68aeccfcdee0279bc2 is 50, key is test_row_0/C:col10/1731435389440/Put/seqid=0 2024-11-12T18:16:31,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742447_1623 (size=12151) 2024-11-12T18:16:31,023 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/36d2c9db34a14a68aeccfcdee0279bc2 2024-11-12T18:16:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/2df9ef720deb43aba64946812e6f9ef4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4 2024-11-12T18:16:31,037 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4, entries=150, sequenceid=193, filesize=30.4 K 2024-11-12T18:16:31,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/97f4e19619cc40c2a2378049392f120d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/97f4e19619cc40c2a2378049392f120d 2024-11-12T18:16:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,049 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/97f4e19619cc40c2a2378049392f120d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T18:16:31,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/36d2c9db34a14a68aeccfcdee0279bc2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36d2c9db34a14a68aeccfcdee0279bc2 2024-11-12T18:16:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36d2c9db34a14a68aeccfcdee0279bc2, entries=150, sequenceid=193, filesize=11.9 K 2024-11-12T18:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,057 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 155ms, sequenceid=193, compaction requested=true 2024-11-12T18:16:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-12T18:16:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-12T18:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T18:16:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1040 sec 2024-11-12T18:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.1100 sec 2024-11-12T18:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T18:16:31,091 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,092 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-12T18:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,095 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T18:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,096 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T18:16:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T18:16:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:31,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-12T18:16:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-12T18:16:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T18:16:31,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-12T18:16:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 163 msec 2024-11-12T18:16:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T18:16:31,411 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-12T18:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T18:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,414 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,415 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:31,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T18:16:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T18:16:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-12T18:16:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-12T18:16:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-12T18:16:31,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-12T18:16:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 164 msec 2024-11-12T18:16:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T18:16:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111294c029dbe3824b6c93e9b0a5726eb6e5_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T18:16:31,731 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:31,732 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-12T18:16:31,735 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T18:16:31,735 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:31,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:31,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742448_1624 (size=19774) 2024-11-12T18:16:31,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111294c029dbe3824b6c93e9b0a5726eb6e5_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111294c029dbe3824b6c93e9b0a5726eb6e5_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:31,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0e196c4d91524a8b878b14e6bb6f7f98, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:31,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0e196c4d91524a8b878b14e6bb6f7f98 is 175, key is test_row_0/A:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:31,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435391722, max timestamp=1731435391759Num uniques 859; 2024-11-12T18:16:31,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:16:31,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435391722, max timestamp=1731435391759Num uniques 859; 2024-11-12T18:16:31,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:16:31,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742449_1625 (size=57033) 2024-11-12T18:16:31,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0e196c4d91524a8b878b14e6bb6f7f98 2024-11-12T18:16:31,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435391722, max timestamp=1731435391764Num uniques 859; 2024-11-12T18:16:31,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T18:16:31,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/2fc7b4f0bcf34d9b8e654ee43753619f is 50, key is test_row_0/B:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:31,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742450_1626 (size=12151) 2024-11-12T18:16:31,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/2fc7b4f0bcf34d9b8e654ee43753619f 2024-11-12T18:16:31,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/f87eae5c409c48b3bfcc33c30f034a5d is 50, key is test_row_0/C:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435451804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:31,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731435391798, max timestamp=1731435391810Num uniques 1018; 2024-11-12T18:16:31,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T18:16:31,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731435391802, max timestamp=1731435391810Num uniques 1018; 2024-11-12T18:16:31,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T18:16:31,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731435391802, max timestamp=1731435391810Num uniques 1009; 2024-11-12T18:16:31,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-12T18:16:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742451_1627 (size=12151) 2024-11-12T18:16:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T18:16:31,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T18:16:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:31,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435451890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:31,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435451891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435451891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435451920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435452000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435452000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435452000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T18:16:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T18:16:32,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435452130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T18:16:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435452210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435452210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435452210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,214 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435452210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/f87eae5c409c48b3bfcc33c30f034a5d 2024-11-12T18:16:32,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0e196c4d91524a8b878b14e6bb6f7f98 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98 2024-11-12T18:16:32,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98, entries=300, sequenceid=205, filesize=55.7 K 2024-11-12T18:16:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/2fc7b4f0bcf34d9b8e654ee43753619f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2fc7b4f0bcf34d9b8e654ee43753619f 2024-11-12T18:16:32,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2fc7b4f0bcf34d9b8e654ee43753619f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T18:16:32,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/f87eae5c409c48b3bfcc33c30f034a5d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f87eae5c409c48b3bfcc33c30f034a5d 2024-11-12T18:16:32,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f87eae5c409c48b3bfcc33c30f034a5d, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T18:16:32,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.14 KB/134292 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 571ms, sequenceid=205, compaction requested=true 2024-11-12T18:16:32,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:32,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:32,244 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:32,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:32,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:32,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:32,244 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:32,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:32,245 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159334 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:32,245 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:32,245 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:32,246 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,246 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:32,246 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,246 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/398be3ed29854bed9168550f38149f7b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=155.6 K 2024-11-12T18:16:32,246 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,246 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/739f60ac0f0c4308b256f9758e058925, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e4b8cd9ae6c54dbea197eaf0e91a26b9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/97f4e19619cc40c2a2378049392f120d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2fc7b4f0bcf34d9b8e654ee43753619f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=47.8 K 2024-11-12T18:16:32,246 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/398be3ed29854bed9168550f38149f7b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98] 2024-11-12T18:16:32,246 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 739f60ac0f0c4308b256f9758e058925, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435387624 2024-11-12T18:16:32,246 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 398be3ed29854bed9168550f38149f7b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435387624 2024-11-12T18:16:32,246 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b8cd9ae6c54dbea197eaf0e91a26b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731435388779 2024-11-12T18:16:32,246 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a5efc8b3b94585a7d9088f35e240e3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731435388779 2024-11-12T18:16:32,247 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f4e19619cc40c2a2378049392f120d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731435389438 2024-11-12T18:16:32,247 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2df9ef720deb43aba64946812e6f9ef4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731435389438 2024-11-12T18:16:32,247 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fc7b4f0bcf34d9b8e654ee43753619f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731435391667 2024-11-12T18:16:32,247 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e196c4d91524a8b878b14e6bb6f7f98, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731435391658 2024-11-12T18:16:32,253 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:32,255 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:32,256 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e69f3831f4744d849f972adc36747fe3 is 50, key is test_row_0/B:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:32,264 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411126fe487695f0644f7a66a38657f5b9408_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:32,266 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411126fe487695f0644f7a66a38657f5b9408_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:32,266 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126fe487695f0644f7a66a38657f5b9408_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742452_1628 (size=12629) 2024-11-12T18:16:32,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742453_1629 (size=4469) 2024-11-12T18:16:32,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T18:16:32,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,352 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=131.14 KB heapSize=372.55 KB 2024-11-12T18:16:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:32,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T18:16:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111260e7fd8d26484c079fabcd6ab4308dc7_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435391889/Put/seqid=0 2024-11-12T18:16:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742454_1630 (size=12354) 2024-11-12T18:16:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:32,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435452481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435452520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435452520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435452520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435452601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,671 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#533 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:32,672 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4f1279a744cb48c1b1c3dbd86744d2b2 is 175, key is test_row_0/A:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:32,673 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/e69f3831f4744d849f972adc36747fe3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e69f3831f4744d849f972adc36747fe3 2024-11-12T18:16:32,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742455_1631 (size=31583) 2024-11-12T18:16:32,681 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into e69f3831f4744d849f972adc36747fe3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:32,681 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:32,681 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=12, startTime=1731435392244; duration=0sec 2024-11-12T18:16:32,681 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:32,681 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:32,681 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T18:16:32,681 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4f1279a744cb48c1b1c3dbd86744d2b2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4f1279a744cb48c1b1c3dbd86744d2b2 2024-11-12T18:16:32,683 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T18:16:32,683 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:32,683 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:32,683 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/be2051941af64cf88fef4df3d2592063, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36d2c9db34a14a68aeccfcdee0279bc2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f87eae5c409c48b3bfcc33c30f034a5d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=47.8 K 2024-11-12T18:16:32,683 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e8c03468bb1e4dcbb0c3e7f9f0d22fbe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731435387624 2024-11-12T18:16:32,684 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting be2051941af64cf88fef4df3d2592063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731435388779 2024-11-12T18:16:32,685 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d2c9db34a14a68aeccfcdee0279bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731435389438 2024-11-12T18:16:32,685 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting f87eae5c409c48b3bfcc33c30f034a5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731435391667 2024-11-12T18:16:32,692 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into 4f1279a744cb48c1b1c3dbd86744d2b2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:32,692 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:32,692 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=12, startTime=1731435392244; duration=0sec 2024-11-12T18:16:32,692 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:32,692 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:32,698 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#536 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:32,698 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/615147057b1440d88e169c18548f5873 is 50, key is test_row_0/C:col10/1731435391671/Put/seqid=0 2024-11-12T18:16:32,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742456_1632 (size=12629) 2024-11-12T18:16:32,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:32,782 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111260e7fd8d26484c079fabcd6ab4308dc7_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260e7fd8d26484c079fabcd6ab4308dc7_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/88381ee198074833b6e2795171962f11, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:32,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/88381ee198074833b6e2795171962f11 is 175, key is test_row_0/A:col10/1731435391889/Put/seqid=0 2024-11-12T18:16:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742457_1633 (size=31155) 2024-11-12T18:16:32,787 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/88381ee198074833b6e2795171962f11 2024-11-12T18:16:32,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/0ed97b0cc3b647099a5bee30b019865a is 50, key is test_row_0/B:col10/1731435391889/Put/seqid=0 2024-11-12T18:16:32,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742458_1634 (size=12201) 2024-11-12T18:16:32,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435452810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T18:16:33,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435453030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:33,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435453030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435453040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:33,105 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/615147057b1440d88e169c18548f5873 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/615147057b1440d88e169c18548f5873 2024-11-12T18:16:33,109 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into 615147057b1440d88e169c18548f5873(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:33,109 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:33,109 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=12, startTime=1731435392244; duration=0sec 2024-11-12T18:16:33,109 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:33,109 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:33,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435453120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:33,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/0ed97b0cc3b647099a5bee30b019865a 2024-11-12T18:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/b41eed0c841a43c4bbea6caa9380059c is 50, key is test_row_0/C:col10/1731435391889/Put/seqid=0 2024-11-12T18:16:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742459_1635 (size=12201) 2024-11-12T18:16:33,226 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/b41eed0c841a43c4bbea6caa9380059c 2024-11-12T18:16:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/88381ee198074833b6e2795171962f11 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11 2024-11-12T18:16:33,232 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11, entries=150, sequenceid=258, filesize=30.4 K 2024-11-12T18:16:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/0ed97b0cc3b647099a5bee30b019865a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/0ed97b0cc3b647099a5bee30b019865a 2024-11-12T18:16:33,236 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/0ed97b0cc3b647099a5bee30b019865a, entries=150, sequenceid=258, filesize=11.9 K 2024-11-12T18:16:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/b41eed0c841a43c4bbea6caa9380059c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b41eed0c841a43c4bbea6caa9380059c 2024-11-12T18:16:33,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b41eed0c841a43c4bbea6caa9380059c, entries=150, sequenceid=258, filesize=11.9 K 2024-11-12T18:16:33,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~131.14 KB/134292, heapSize ~372.50 KB/381440, currentSize=46.96 KB/48090 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 888ms, sequenceid=258, compaction requested=false 2024-11-12T18:16:33,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:33,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:33,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-12T18:16:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-12T18:16:33,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-12T18:16:33,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5060 sec 2024-11-12T18:16:33,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.5120 sec 2024-11-12T18:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:33,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T18:16:33,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:33,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:33,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:33,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:33,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:33,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:33,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111251b0ba1318d841bd9e6fef18e967e487_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:33,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742460_1636 (size=12454) 2024-11-12T18:16:33,673 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:33,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111251b0ba1318d841bd9e6fef18e967e487_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111251b0ba1318d841bd9e6fef18e967e487_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:33,686 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/259f5cf69e924b279cacd343d62a75ea, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:33,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/259f5cf69e924b279cacd343d62a75ea is 175, key is test_row_0/A:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:33,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742461_1637 (size=31255) 2024-11-12T18:16:33,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435393713, max timestamp=1731435393741Num uniques 859; 2024-11-12T18:16:33,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:16:33,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435393713, max timestamp=1731435393741Num uniques 859; 2024-11-12T18:16:33,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:16:33,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731435393713, max timestamp=1731435393741Num uniques 859; 2024-11-12T18:16:33,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9911683f163c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T18:16:33,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435453831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T18:16:33,880 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:33,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-12T18:16:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T18:16:33,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:33,883 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:33,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:33,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435453941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T18:16:34,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T18:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435454041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:34,045 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:34,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435454050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:34,054 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_2' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:34,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435454060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:34,065 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:34,092 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/259f5cf69e924b279cacd343d62a75ea 2024-11-12T18:16:34,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/18a450df36b54cc9b5c3323844207694 is 50, key is test_row_0/B:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:34,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742462_1638 (size=12301) 2024-11-12T18:16:34,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/18a450df36b54cc9b5c3323844207694 2024-11-12T18:16:34,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/a6565b0e07eb463eb196dbe6576b5f99 is 50, key is test_row_0/C:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:34,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742463_1639 (size=12301) 2024-11-12T18:16:34,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435454150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:34,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T18:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T18:16:34,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T18:16:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435454460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:34,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T18:16:34,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T18:16:34,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/a6565b0e07eb463eb196dbe6576b5f99 2024-11-12T18:16:34,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/259f5cf69e924b279cacd343d62a75ea as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea 2024-11-12T18:16:34,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea, entries=150, sequenceid=272, filesize=30.5 K 2024-11-12T18:16:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/18a450df36b54cc9b5c3323844207694 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/18a450df36b54cc9b5c3323844207694 2024-11-12T18:16:34,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/18a450df36b54cc9b5c3323844207694, entries=150, sequenceid=272, filesize=12.0 K 2024-11-12T18:16:34,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/a6565b0e07eb463eb196dbe6576b5f99 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/a6565b0e07eb463eb196dbe6576b5f99 2024-11-12T18:16:34,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/a6565b0e07eb463eb196dbe6576b5f99, entries=150, sequenceid=272, filesize=12.0 K 2024-11-12T18:16:34,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 905ms, sequenceid=272, compaction requested=true 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:34,538 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:34,538 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:34,539 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:34,539 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:34,539 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,539 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4f1279a744cb48c1b1c3dbd86744d2b2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=91.8 K 2024-11-12T18:16:34,539 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,539 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4f1279a744cb48c1b1c3dbd86744d2b2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea] 2024-11-12T18:16:34,539 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37131 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:34,540 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f1279a744cb48c1b1c3dbd86744d2b2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731435391667 2024-11-12T18:16:34,540 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e69f3831f4744d849f972adc36747fe3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/0ed97b0cc3b647099a5bee30b019865a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/18a450df36b54cc9b5c3323844207694] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=36.3 K 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88381ee198074833b6e2795171962f11, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731435391888 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting e69f3831f4744d849f972adc36747fe3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731435391667 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259f5cf69e924b279cacd343d62a75ea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731435392470 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed97b0cc3b647099a5bee30b019865a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731435391888 2024-11-12T18:16:34,540 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a450df36b54cc9b5c3323844207694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731435392470 2024-11-12T18:16:34,545 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:34,546 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#542 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:34,547 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/dde03ae8a3d84b7995d5845f2258b890 is 50, key is test_row_0/B:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:34,549 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111257d664bfed9c438bac31328191f27804_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:34,551 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111257d664bfed9c438bac31328191f27804_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:34,551 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111257d664bfed9c438bac31328191f27804_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742464_1640 (size=12881) 2024-11-12T18:16:34,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742465_1641 (size=4469) 2024-11-12T18:16:34,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T18:16:34,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124b5d80a314c7400d8f11cf2dfe89fa0f_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435393823/Put/seqid=0 2024-11-12T18:16:34,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742466_1642 (size=12454) 2024-11-12T18:16:34,963 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#543 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:34,964 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/9545b07512514dd5a4a1217dd50a86dc is 175, key is test_row_0/A:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:34,978 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/dde03ae8a3d84b7995d5845f2258b890 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/dde03ae8a3d84b7995d5845f2258b890 2024-11-12T18:16:34,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:34,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742467_1643 (size=31835) 2024-11-12T18:16:34,993 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into dde03ae8a3d84b7995d5845f2258b890(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:34,993 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:34,993 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=13, startTime=1731435394538; duration=0sec 2024-11-12T18:16:34,993 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:34,993 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:34,993 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:34,997 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37131 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:34,997 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:34,998 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:34,998 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/615147057b1440d88e169c18548f5873, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b41eed0c841a43c4bbea6caa9380059c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/a6565b0e07eb463eb196dbe6576b5f99] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=36.3 K 2024-11-12T18:16:34,998 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 615147057b1440d88e169c18548f5873, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731435391667 2024-11-12T18:16:34,999 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting b41eed0c841a43c4bbea6caa9380059c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731435391888 2024-11-12T18:16:35,000 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/9545b07512514dd5a4a1217dd50a86dc as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/9545b07512514dd5a4a1217dd50a86dc 2024-11-12T18:16:35,003 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a6565b0e07eb463eb196dbe6576b5f99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731435392470 2024-11-12T18:16:35,015 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:35,016 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/54623c654c824fd0abef0765c1a7b311 is 50, key is test_row_0/C:col10/1731435393632/Put/seqid=0 2024-11-12T18:16:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T18:16:35,028 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into 9545b07512514dd5a4a1217dd50a86dc(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:35,028 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:35,028 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=13, startTime=1731435394538; duration=0sec 2024-11-12T18:16:35,029 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:35,029 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:35,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742468_1644 (size=12881) 2024-11-12T18:16:35,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:35,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435455064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:35,087 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/54623c654c824fd0abef0765c1a7b311 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/54623c654c824fd0abef0765c1a7b311 2024-11-12T18:16:35,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124b5d80a314c7400d8f11cf2dfe89fa0f_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124b5d80a314c7400d8f11cf2dfe89fa0f_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/bb54c33964a04227bb8dc2235615c71e, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:35,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/bb54c33964a04227bb8dc2235615c71e is 175, key is test_row_0/A:col10/1731435393823/Put/seqid=0 2024-11-12T18:16:35,117 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into 54623c654c824fd0abef0765c1a7b311(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:35,117 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:35,117 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=13, startTime=1731435394538; duration=0sec 2024-11-12T18:16:35,117 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:35,117 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742469_1645 (size=31255) 2024-11-12T18:16:35,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435455181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435455391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:35,541 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/bb54c33964a04227bb8dc2235615c71e 2024-11-12T18:16:35,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/19322806618f48d7aa7c2a3c4fb88051 is 50, key is test_row_0/B:col10/1731435393823/Put/seqid=0 2024-11-12T18:16:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742470_1646 (size=12301) 2024-11-12T18:16:35,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/19322806618f48d7aa7c2a3c4fb88051 2024-11-12T18:16:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/fd0990c4f3044d63845e741bc434dbb8 is 50, key is test_row_0/C:col10/1731435393823/Put/seqid=0 2024-11-12T18:16:35,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742471_1647 (size=12301) 2024-11-12T18:16:35,689 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/fd0990c4f3044d63845e741bc434dbb8 2024-11-12T18:16:35,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435455700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/bb54c33964a04227bb8dc2235615c71e as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e 2024-11-12T18:16:35,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e, entries=150, sequenceid=308, filesize=30.5 K 2024-11-12T18:16:35,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/19322806618f48d7aa7c2a3c4fb88051 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/19322806618f48d7aa7c2a3c4fb88051 2024-11-12T18:16:35,746 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/19322806618f48d7aa7c2a3c4fb88051, entries=150, sequenceid=308, filesize=12.0 K 2024-11-12T18:16:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/fd0990c4f3044d63845e741bc434dbb8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/fd0990c4f3044d63845e741bc434dbb8 2024-11-12T18:16:35,777 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/fd0990c4f3044d63845e741bc434dbb8, entries=150, sequenceid=308, filesize=12.0 K 2024-11-12T18:16:35,781 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1133ms, sequenceid=308, compaction requested=false 2024-11-12T18:16:35,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:35,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:35,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-12T18:16:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-12T18:16:35,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-12T18:16:35,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9020 sec 2024-11-12T18:16:35,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9090 sec 2024-11-12T18:16:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T18:16:36,037 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:36,038 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-12T18:16:36,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:36,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:36,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:36,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T18:16:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:36,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b35de83e571c4e76a40a9eb0ebf75b43_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:36,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742472_1648 (size=12454) 2024-11-12T18:16:36,140 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435456135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435456135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:36,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435456145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,174 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b35de83e571c4e76a40a9eb0ebf75b43_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b35de83e571c4e76a40a9eb0ebf75b43_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:36,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/b73dea59a50140479af31a123cbdb794, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:36,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/b73dea59a50140479af31a123cbdb794 is 175, key is test_row_0/A:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:36,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T18:16:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:36,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:36,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742473_1649 (size=31255) 2024-11-12T18:16:36,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435456224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,228 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/b73dea59a50140479af31a123cbdb794 2024-11-12T18:16:36,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435456252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435456253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/3ec967c5042b4a90af83585e6437c78c is 50, key is test_row_0/B:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:36,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435456261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742474_1650 (size=12301) 2024-11-12T18:16:36,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/3ec967c5042b4a90af83585e6437c78c 2024-11-12T18:16:36,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/6a8446568e8742078c72f090fb88a07d is 50, key is test_row_0/C:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:36,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T18:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:36,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742475_1651 (size=12301) 2024-11-12T18:16:36,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/6a8446568e8742078c72f090fb88a07d 2024-11-12T18:16:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/b73dea59a50140479af31a123cbdb794 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794 2024-11-12T18:16:36,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794, entries=150, sequenceid=323, filesize=30.5 K 2024-11-12T18:16:36,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435456461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/3ec967c5042b4a90af83585e6437c78c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/3ec967c5042b4a90af83585e6437c78c 2024-11-12T18:16:36,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435456465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435456475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/3ec967c5042b4a90af83585e6437c78c, entries=150, sequenceid=323, filesize=12.0 K 2024-11-12T18:16:36,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/6a8446568e8742078c72f090fb88a07d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/6a8446568e8742078c72f090fb88a07d 2024-11-12T18:16:36,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/6a8446568e8742078c72f090fb88a07d, entries=150, sequenceid=323, filesize=12.0 K 2024-11-12T18:16:36,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 441ms, sequenceid=323, compaction requested=true 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T18:16:36,514 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:36,514 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:36,515 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:36,515 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:36,515 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:36,515 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:36,515 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,516 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,516 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/54623c654c824fd0abef0765c1a7b311, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/fd0990c4f3044d63845e741bc434dbb8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/6a8446568e8742078c72f090fb88a07d] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=36.6 K 2024-11-12T18:16:36,516 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/9545b07512514dd5a4a1217dd50a86dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=92.1 K 2024-11-12T18:16:36,516 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,516 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/9545b07512514dd5a4a1217dd50a86dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794] 2024-11-12T18:16:36,516 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 54623c654c824fd0abef0765c1a7b311, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731435392470 2024-11-12T18:16:36,516 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9545b07512514dd5a4a1217dd50a86dc, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731435392470 2024-11-12T18:16:36,516 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0990c4f3044d63845e741bc434dbb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731435393758 2024-11-12T18:16:36,517 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8446568e8742078c72f090fb88a07d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731435395017 2024-11-12T18:16:36,517 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb54c33964a04227bb8dc2235615c71e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731435393758 2024-11-12T18:16:36,518 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting b73dea59a50140479af31a123cbdb794, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731435395017 2024-11-12T18:16:36,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,525 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:36,532 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#551 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:36,532 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/ea4f061a9c4e43808dca2237eaad2896 is 50, key is test_row_0/C:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:36,537 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:36,555 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d28e778538224695bbd9174cb54ae281_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:36,557 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d28e778538224695bbd9174cb54ae281_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:36,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742476_1652 (size=12983) 2024-11-12T18:16:36,558 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d28e778538224695bbd9174cb54ae281_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:36,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f386b2a3c4c64c6bb01c165ba3d82d32_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:36,574 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/ea4f061a9c4e43808dca2237eaad2896 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ea4f061a9c4e43808dca2237eaad2896 2024-11-12T18:16:36,582 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into ea4f061a9c4e43808dca2237eaad2896(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:36,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:36,583 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=13, startTime=1731435396514; duration=0sec 2024-11-12T18:16:36,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:36,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:36,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:36,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742477_1653 (size=12454) 2024-11-12T18:16:36,584 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:36,584 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:36,584 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:36,585 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/dde03ae8a3d84b7995d5845f2258b890, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/19322806618f48d7aa7c2a3c4fb88051, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/3ec967c5042b4a90af83585e6437c78c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=36.6 K 2024-11-12T18:16:36,592 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting dde03ae8a3d84b7995d5845f2258b890, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731435392470 2024-11-12T18:16:36,593 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 19322806618f48d7aa7c2a3c4fb88051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731435393758 2024-11-12T18:16:36,597 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec967c5042b4a90af83585e6437c78c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731435395017 2024-11-12T18:16:36,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742478_1654 (size=4469) 2024-11-12T18:16:36,663 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:36,663 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/a9351fb0f5a04f81877a45ce8f5b65b2 is 50, key is test_row_0/B:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:36,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742479_1655 (size=12983) 2024-11-12T18:16:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:36,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435456822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435456823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435456824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435456940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435456941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435456944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:36,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:37,008 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f386b2a3c4c64c6bb01c165ba3d82d32_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f386b2a3c4c64c6bb01c165ba3d82d32_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:37,017 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#552 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:37,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/45b1960258e544708abd3f4d146335f7, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:37,017 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/dc332165c5c442f9a20bccc9b90108a4 is 175, key is test_row_0/A:col10/1731435395017/Put/seqid=0 2024-11-12T18:16:37,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/45b1960258e544708abd3f4d146335f7 is 175, key is test_row_0/A:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:37,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742481_1657 (size=31255) 2024-11-12T18:16:37,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742480_1656 (size=31937) 2024-11-12T18:16:37,091 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/dc332165c5c442f9a20bccc9b90108a4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/dc332165c5c442f9a20bccc9b90108a4 2024-11-12T18:16:37,115 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into dc332165c5c442f9a20bccc9b90108a4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:37,115 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:37,115 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=13, startTime=1731435396514; duration=0sec 2024-11-12T18:16:37,115 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:37,115 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:37,124 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/a9351fb0f5a04f81877a45ce8f5b65b2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a9351fb0f5a04f81877a45ce8f5b65b2 2024-11-12T18:16:37,145 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into a9351fb0f5a04f81877a45ce8f5b65b2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:37,145 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:37,145 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=13, startTime=1731435396514; duration=0sec 2024-11-12T18:16:37,145 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:37,145 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435457151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435457152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435457163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:37,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435457247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,257 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435457464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,473 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/45b1960258e544708abd3f4d146335f7 2024-11-12T18:16:37,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435457471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435457473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/ee91982cc12d4b8db86654f62f8a82a4 is 50, key is test_row_0/B:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742482_1658 (size=12301) 2024-11-12T18:16:37,581 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/ee91982cc12d4b8db86654f62f8a82a4 2024-11-12T18:16:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/5e1fcbe2ca614a06add39db9ed7e96f1 is 50, key is test_row_0/C:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:37,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742483_1659 (size=12301) 2024-11-12T18:16:37,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435457980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435457981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:37,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435457991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:38,108 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/5e1fcbe2ca614a06add39db9ed7e96f1 2024-11-12T18:16:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/45b1960258e544708abd3f4d146335f7 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7 2024-11-12T18:16:38,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7, entries=150, sequenceid=343, filesize=30.5 K 2024-11-12T18:16:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/ee91982cc12d4b8db86654f62f8a82a4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ee91982cc12d4b8db86654f62f8a82a4 2024-11-12T18:16:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,221 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ee91982cc12d4b8db86654f62f8a82a4, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T18:16:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/5e1fcbe2ca614a06add39db9ed7e96f1 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/5e1fcbe2ca614a06add39db9ed7e96f1 2024-11-12T18:16:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,245 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/5e1fcbe2ca614a06add39db9ed7e96f1, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T18:16:38,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1721ms, sequenceid=343, compaction requested=false 2024-11-12T18:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-12T18:16:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-12T18:16:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-12T18:16:38,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2070 sec 2024-11-12T18:16:38,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.2240 sec 2024-11-12T18:16:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T18:16:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b0eac5ba3816467bb1c1a6516cb1c677_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_1/A:col10/1731435396817/Put/seqid=0 2024-11-12T18:16:39,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435459094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435459095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435459098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742484_1660 (size=14994) 2024-11-12T18:16:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435459214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435459214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435459223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59388 deadline: 1731435459270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,271 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_1' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435459430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435459437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435459441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,525 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:39,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b0eac5ba3816467bb1c1a6516cb1c677_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b0eac5ba3816467bb1c1a6516cb1c677_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:39,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0101e93e875f4dadab1ce0166454769c, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:39,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0101e93e875f4dadab1ce0166454769c is 175, key is test_row_1/A:col10/1731435396817/Put/seqid=0 2024-11-12T18:16:39,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742485_1661 (size=39945) 2024-11-12T18:16:39,629 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0101e93e875f4dadab1ce0166454769c 2024-11-12T18:16:39,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/bf84ae42657645778c3c11eeadd9ca68 is 50, key is test_row_1/B:col10/1731435396817/Put/seqid=0 2024-11-12T18:16:39,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742486_1662 (size=9857) 2024-11-12T18:16:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435459752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435459753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435459753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:39,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:40,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/bf84ae42657645778c3c11eeadd9ca68 2024-11-12T18:16:40,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/3d27e5004e3042e089405d2532855c94 is 50, key is test_row_1/C:col10/1731435396817/Put/seqid=0 2024-11-12T18:16:40,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742487_1663 (size=9857) 2024-11-12T18:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-12T18:16:40,203 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:40,205 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T18:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-12T18:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:40,209 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T18:16:40,217 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T18:16:40,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T18:16:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435460274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:40,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435460274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:40,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435460274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:40,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:40,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-12T18:16:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:40,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:40,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-12T18:16:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:40,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/3d27e5004e3042e089405d2532855c94 2024-11-12T18:16:40,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/0101e93e875f4dadab1ce0166454769c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c 2024-11-12T18:16:40,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-12T18:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T18:16:40,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c, entries=200, sequenceid=360, filesize=39.0 K 2024-11-12T18:16:40,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/bf84ae42657645778c3c11eeadd9ca68 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/bf84ae42657645778c3c11eeadd9ca68 2024-11-12T18:16:40,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/bf84ae42657645778c3c11eeadd9ca68, entries=100, sequenceid=360, filesize=9.6 K 2024-11-12T18:16:40,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/3d27e5004e3042e089405d2532855c94 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3d27e5004e3042e089405d2532855c94 2024-11-12T18:16:40,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3d27e5004e3042e089405d2532855c94, entries=100, sequenceid=360, filesize=9.6 K 2024-11-12T18:16:40,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1793ms, sequenceid=360, compaction requested=true 2024-11-12T18:16:40,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:40,809 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:40,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:16:40,812 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:40,825 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103137 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:40,825 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:40,825 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,825 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/dc332165c5c442f9a20bccc9b90108a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=100.7 K 2024-11-12T18:16:40,825 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,825 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/dc332165c5c442f9a20bccc9b90108a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c] 2024-11-12T18:16:40,826 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc332165c5c442f9a20bccc9b90108a4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731435395017 2024-11-12T18:16:40,826 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b1960258e544708abd3f4d146335f7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731435396124 2024-11-12T18:16:40,826 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:40,826 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:40,826 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,826 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a9351fb0f5a04f81877a45ce8f5b65b2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ee91982cc12d4b8db86654f62f8a82a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/bf84ae42657645778c3c11eeadd9ca68] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=34.3 K 2024-11-12T18:16:40,826 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0101e93e875f4dadab1ce0166454769c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731435396817 2024-11-12T18:16:40,832 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting a9351fb0f5a04f81877a45ce8f5b65b2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731435395017 2024-11-12T18:16:40,834 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ee91982cc12d4b8db86654f62f8a82a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731435396124 2024-11-12T18:16:40,835 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting bf84ae42657645778c3c11eeadd9ca68, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731435396817 2024-11-12T18:16:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:40,855 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:40,859 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#561 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:40,860 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/c58cd248c41348cc95328f339c8646e8 is 50, key is test_row_0/B:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:40,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43757 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:40,863 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:40,871 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111254498eede8674febb26809570fa30a7f_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:40,875 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111254498eede8674febb26809570fa30a7f_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:40,875 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111254498eede8674febb26809570fa30a7f_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:40,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122409f25fd7f242968b0110521e821ec4_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435399018/Put/seqid=0 2024-11-12T18:16:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742489_1665 (size=4469) 2024-11-12T18:16:40,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742488_1664 (size=13085) 2024-11-12T18:16:40,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742490_1666 (size=12454) 2024-11-12T18:16:40,981 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/c58cd248c41348cc95328f339c8646e8 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/c58cd248c41348cc95328f339c8646e8 2024-11-12T18:16:41,004 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into c58cd248c41348cc95328f339c8646e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:41,004 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:41,004 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=13, startTime=1731435400810; duration=0sec 2024-11-12T18:16:41,004 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:41,004 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:41,004 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:41,008 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:41,009 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:41,009 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:41,009 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ea4f061a9c4e43808dca2237eaad2896, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/5e1fcbe2ca614a06add39db9ed7e96f1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3d27e5004e3042e089405d2532855c94] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=34.3 K 2024-11-12T18:16:41,013 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4f061a9c4e43808dca2237eaad2896, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731435395017 2024-11-12T18:16:41,023 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1fcbe2ca614a06add39db9ed7e96f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731435396124 2024-11-12T18:16:41,029 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d27e5004e3042e089405d2532855c94, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731435396817 2024-11-12T18:16:41,055 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#563 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:41,055 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/3c5a8466c41d4d9cafa83d9fac7ba4f4 is 50, key is test_row_0/C:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:41,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742491_1667 (size=13085) 2024-11-12T18:16:41,123 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/3c5a8466c41d4d9cafa83d9fac7ba4f4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3c5a8466c41d4d9cafa83d9fac7ba4f4 2024-11-12T18:16:41,153 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into 3c5a8466c41d4d9cafa83d9fac7ba4f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:41,153 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:41,153 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=13, startTime=1731435400810; duration=0sec 2024-11-12T18:16:41,153 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:41,153 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:41,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. as already flushing 2024-11-12T18:16:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:41,329 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#560 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:41,330 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/cdb2f6cc3f94428992073f2b47b1e257 is 175, key is test_row_0/A:col10/1731435396128/Put/seqid=0 2024-11-12T18:16:41,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435461334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435461340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435461343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:41,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742492_1668 (size=32146) 2024-11-12T18:16:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:41,390 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/cdb2f6cc3f94428992073f2b47b1e257 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/cdb2f6cc3f94428992073f2b47b1e257 2024-11-12T18:16:41,401 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into cdb2f6cc3f94428992073f2b47b1e257(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:41,401 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:41,401 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=13, startTime=1731435400809; duration=0sec 2024-11-12T18:16:41,401 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:41,401 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:41,402 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122409f25fd7f242968b0110521e821ec4_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122409f25fd7f242968b0110521e821ec4_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:41,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4c278239415d4dbdb9a659af1f5f2a5d, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:41,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4c278239415d4dbdb9a659af1f5f2a5d is 175, key is test_row_0/A:col10/1731435399018/Put/seqid=0 2024-11-12T18:16:41,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742493_1669 (size=31255) 2024-11-12T18:16:41,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435461450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435461450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435461460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,613 INFO [Thread-2550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2550. 2024-11-12T18:16:41,613 DEBUG [Thread-2550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:41,613 DEBUG [Thread-2550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,614 DEBUG [Thread-2550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:41,617 INFO [Thread-2544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2544. 2024-11-12T18:16:41,617 DEBUG [Thread-2544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:41,617 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,617 DEBUG [Thread-2544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:41,618 INFO [Thread-2548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2548. 2024-11-12T18:16:41,618 DEBUG [Thread-2548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:41,618 DEBUG [Thread-2548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:41,618 INFO [Thread-2546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2546. 2024-11-12T18:16:41,618 DEBUG [Thread-2548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,618 DEBUG [Thread-2546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:41,618 DEBUG [Thread-2546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,619 DEBUG [Thread-2546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:41,621 INFO [Thread-2542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2542. 2024-11-12T18:16:41,621 DEBUG [Thread-2542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:41,621 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,622 DEBUG [Thread-2542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:41,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435461673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,843 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4c278239415d4dbdb9a659af1f5f2a5d 2024-11-12T18:16:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/a4ff2f1c09a94684a564133f82ac072a is 50, key is test_row_0/B:col10/1731435399018/Put/seqid=0 2024-11-12T18:16:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742494_1670 (size=12301) 2024-11-12T18:16:41,927 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/a4ff2f1c09a94684a564133f82ac072a 2024-11-12T18:16:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e8cc7320a7874d65b68bf6a3c9f8a011 is 50, key is test_row_0/C:col10/1731435399018/Put/seqid=0 2024-11-12T18:16:41,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59428 deadline: 1731435461984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59414 deadline: 1731435461985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:41,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59382 deadline: 1731435461997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:42,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:42,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742495_1671 (size=12301) 2024-11-12T18:16:42,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T18:16:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43757 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59392 deadline: 1731435462220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 2024-11-12T18:16:42,221 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9911683f163c,43757,1731435233300 for 'test_row_0' in ffab90d6ee04bb318f8ea8ffd27a4f4b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T18:16:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 , the old value is region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T18:16:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., hostname=9911683f163c,43757,1731435233300, seqNum=5 because the exception is null or not the one we care about 2024-11-12T18:16:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:42,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e8cc7320a7874d65b68bf6a3c9f8a011 2024-11-12T18:16:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/4c278239415d4dbdb9a659af1f5f2a5d as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d 2024-11-12T18:16:42,465 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d, entries=150, sequenceid=378, filesize=30.5 K 2024-11-12T18:16:42,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/a4ff2f1c09a94684a564133f82ac072a as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a4ff2f1c09a94684a564133f82ac072a 2024-11-12T18:16:42,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a4ff2f1c09a94684a564133f82ac072a, entries=150, sequenceid=378, filesize=12.0 K 2024-11-12T18:16:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/e8cc7320a7874d65b68bf6a3c9f8a011 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8cc7320a7874d65b68bf6a3c9f8a011 2024-11-12T18:16:42,480 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8cc7320a7874d65b68bf6a3c9f8a011, entries=150, sequenceid=378, filesize=12.0 K 2024-11-12T18:16:42,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1618ms, sequenceid=378, compaction requested=false 2024-11-12T18:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:42,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9911683f163c:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-12T18:16:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-12T18:16:42,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-12T18:16:42,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2660 sec 2024-11-12T18:16:42,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.2820 sec 2024-11-12T18:16:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43757 {}] regionserver.HRegion(8855): Flush requested on ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:42,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T18:16:42,495 INFO [Thread-2533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2533. 2024-11-12T18:16:42,495 DEBUG [Thread-2533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:42,495 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:42,495 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:42,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:42,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:42,498 INFO [Thread-2535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2535. 2024-11-12T18:16:42,498 DEBUG [Thread-2535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:42,498 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:42,498 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:42,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:42,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c9462ff1978549a6a8a2125abed11440_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:42,516 INFO [Thread-2539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2539. 2024-11-12T18:16:42,516 DEBUG [Thread-2539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:42,516 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:42,516 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:42,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:42,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742496_1672 (size=12454) 2024-11-12T18:16:42,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:42,527 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c9462ff1978549a6a8a2125abed11440_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c9462ff1978549a6a8a2125abed11440_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:42,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/48c5bf2c18f24fc699fb2d62756efc4c, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:42,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/48c5bf2c18f24fc699fb2d62756efc4c is 175, key is test_row_0/A:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:42,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742497_1673 (size=31255) 2024-11-12T18:16:42,962 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/48c5bf2c18f24fc699fb2d62756efc4c 2024-11-12T18:16:42,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/1d0f923e93d344fbb3fc71a6cd777643 is 50, key is test_row_0/B:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:42,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742498_1674 (size=12301) 2024-11-12T18:16:43,313 INFO [Thread-2537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2537. 2024-11-12T18:16:43,313 DEBUG [Thread-2537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:43,314 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:43,314 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:43,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:43,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/1d0f923e93d344fbb3fc71a6cd777643 2024-11-12T18:16:43,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/dbdfc272c07f400395f1081c8ea6bd2f is 50, key is test_row_0/C:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:43,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742499_1675 (size=12301) 2024-11-12T18:16:43,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/dbdfc272c07f400395f1081c8ea6bd2f 2024-11-12T18:16:43,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/48c5bf2c18f24fc699fb2d62756efc4c as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c 2024-11-12T18:16:43,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c, entries=150, sequenceid=397, filesize=30.5 K 2024-11-12T18:16:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/1d0f923e93d344fbb3fc71a6cd777643 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/1d0f923e93d344fbb3fc71a6cd777643 2024-11-12T18:16:43,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/1d0f923e93d344fbb3fc71a6cd777643, entries=150, sequenceid=397, filesize=12.0 K 2024-11-12T18:16:43,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/dbdfc272c07f400395f1081c8ea6bd2f as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/dbdfc272c07f400395f1081c8ea6bd2f 2024-11-12T18:16:43,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/dbdfc272c07f400395f1081c8ea6bd2f, entries=150, sequenceid=397, filesize=12.0 K 2024-11-12T18:16:43,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 1062ms, sequenceid=397, compaction requested=true 2024-11-12T18:16:43,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:43,556 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T18:16:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T18:16:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ffab90d6ee04bb318f8ea8ffd27a4f4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T18:16:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T18:16:43,556 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:43,569 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94656 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:43,569 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/A is initiating minor compaction (all files) 2024-11-12T18:16:43,569 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/A in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:43,569 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/cdb2f6cc3f94428992073f2b47b1e257, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=92.4 K 2024-11-12T18:16:43,569 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:43,569 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. files: [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/cdb2f6cc3f94428992073f2b47b1e257, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c] 2024-11-12T18:16:43,570 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:43,570 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/B is initiating minor compaction (all files) 2024-11-12T18:16:43,570 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/B in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:43,570 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/c58cd248c41348cc95328f339c8646e8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a4ff2f1c09a94684a564133f82ac072a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/1d0f923e93d344fbb3fc71a6cd777643] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=36.8 K 2024-11-12T18:16:43,573 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58cd248c41348cc95328f339c8646e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731435396128 2024-11-12T18:16:43,574 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb2f6cc3f94428992073f2b47b1e257, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731435396128 2024-11-12T18:16:43,575 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ff2f1c09a94684a564133f82ac072a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731435399018 2024-11-12T18:16:43,581 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0f923e93d344fbb3fc71a6cd777643, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731435401332 2024-11-12T18:16:43,581 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c278239415d4dbdb9a659af1f5f2a5d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731435399018 2024-11-12T18:16:43,583 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c5bf2c18f24fc699fb2d62756efc4c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731435401332 2024-11-12T18:16:43,613 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#B#compaction#569 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:43,614 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/adc217607eea4949833df47fa2a55256 is 50, key is test_row_0/B:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:43,637 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:43,645 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112811ade711ca34c7bb04974b9ea0beb02_ffab90d6ee04bb318f8ea8ffd27a4f4b store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:43,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742500_1676 (size=13187) 2024-11-12T18:16:43,657 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112811ade711ca34c7bb04974b9ea0beb02_ffab90d6ee04bb318f8ea8ffd27a4f4b, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:43,657 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112811ade711ca34c7bb04974b9ea0beb02_ffab90d6ee04bb318f8ea8ffd27a4f4b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:43,681 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/adc217607eea4949833df47fa2a55256 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/adc217607eea4949833df47fa2a55256 2024-11-12T18:16:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742501_1677 (size=4469) 2024-11-12T18:16:43,701 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/B of ffab90d6ee04bb318f8ea8ffd27a4f4b into adc217607eea4949833df47fa2a55256(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:43,701 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:43,701 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/B, priority=13, startTime=1731435403556; duration=0sec 2024-11-12T18:16:43,701 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T18:16:43,701 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:B 2024-11-12T18:16:43,702 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T18:16:43,710 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T18:16:43,710 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1541): ffab90d6ee04bb318f8ea8ffd27a4f4b/C is initiating minor compaction (all files) 2024-11-12T18:16:43,710 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ffab90d6ee04bb318f8ea8ffd27a4f4b/C in TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:43,710 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3c5a8466c41d4d9cafa83d9fac7ba4f4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8cc7320a7874d65b68bf6a3c9f8a011, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/dbdfc272c07f400395f1081c8ea6bd2f] into tmpdir=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp, totalSize=36.8 K 2024-11-12T18:16:43,711 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5a8466c41d4d9cafa83d9fac7ba4f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731435396128 2024-11-12T18:16:43,711 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8cc7320a7874d65b68bf6a3c9f8a011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731435399018 2024-11-12T18:16:43,711 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#A#compaction#570 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:43,712 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/300ad5cf230b4d63bc1e9a41c7571492 is 175, key is test_row_0/A:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:43,713 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbdfc272c07f400395f1081c8ea6bd2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731435401332 2024-11-12T18:16:43,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742502_1678 (size=32141) 2024-11-12T18:16:43,764 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ffab90d6ee04bb318f8ea8ffd27a4f4b#C#compaction#571 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T18:16:43,764 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/500abbc978cd4c109e9f162be2cbbe59 is 50, key is test_row_0/C:col10/1731435401332/Put/seqid=0 2024-11-12T18:16:43,792 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/300ad5cf230b4d63bc1e9a41c7571492 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/300ad5cf230b4d63bc1e9a41c7571492 2024-11-12T18:16:43,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742503_1679 (size=13187) 2024-11-12T18:16:43,821 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/A of ffab90d6ee04bb318f8ea8ffd27a4f4b into 300ad5cf230b4d63bc1e9a41c7571492(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:43,821 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:43,821 INFO [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/A, priority=13, startTime=1731435403555; duration=0sec 2024-11-12T18:16:43,821 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:43,821 DEBUG [RS:0;9911683f163c:43757-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:A 2024-11-12T18:16:44,219 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/500abbc978cd4c109e9f162be2cbbe59 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/500abbc978cd4c109e9f162be2cbbe59 2024-11-12T18:16:44,224 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ffab90d6ee04bb318f8ea8ffd27a4f4b/C of ffab90d6ee04bb318f8ea8ffd27a4f4b into 500abbc978cd4c109e9f162be2cbbe59(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T18:16:44,224 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: 2024-11-12T18:16:44,224 INFO [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., storeName=ffab90d6ee04bb318f8ea8ffd27a4f4b/C, priority=13, startTime=1731435403556; duration=0sec 2024-11-12T18:16:44,224 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T18:16:44,224 DEBUG [RS:0;9911683f163c:43757-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ffab90d6ee04bb318f8ea8ffd27a4f4b:C 2024-11-12T18:16:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T18:16:44,371 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:51,243 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T18:16:52,313 INFO [Thread-2531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2531. 2024-11-12T18:16:52,313 DEBUG [Thread-2531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T18:16:52,313 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:52,313 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:52,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2247 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2137 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2127 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2274 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2127 2024-11-12T18:16:52,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T18:16:52,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:16:52,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:16:52,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:52,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:52,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:52,315 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T18:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T18:16:52,319 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435412318"}]},"ts":"1731435412318"} 2024-11-12T18:16:52,320 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T18:16:52,320 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T18:16:52,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T18:16:52,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, UNASSIGN}] 2024-11-12T18:16:52,323 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, UNASSIGN 2024-11-12T18:16:52,324 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=CLOSING, regionLocation=9911683f163c,43757,1731435233300 2024-11-12T18:16:52,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, UNASSIGN because future has completed 2024-11-12T18:16:52,326 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T18:16:52,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300}] 2024-11-12T18:16:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T18:16:52,478 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:52,478 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing ffab90d6ee04bb318f8ea8ffd27a4f4b, disabling compactions & flushes 2024-11-12T18:16:52,479 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. after waiting 0 ms 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:52,479 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=A 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=B 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ffab90d6ee04bb318f8ea8ffd27a4f4b, store=C 2024-11-12T18:16:52,479 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T18:16:52,484 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fd18dcc04f94422fa675eaf4c39eba1d_ffab90d6ee04bb318f8ea8ffd27a4f4b is 50, key is test_row_0/A:col10/1731435412311/Put/seqid=0 2024-11-12T18:16:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742504_1680 (size=9914) 2024-11-12T18:16:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T18:16:52,890 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T18:16:52,894 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fd18dcc04f94422fa675eaf4c39eba1d_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fd18dcc04f94422fa675eaf4c39eba1d_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:52,894 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/60183d34a1c24cd897d1c746939ff868, store: [table=TestAcidGuarantees family=A region=ffab90d6ee04bb318f8ea8ffd27a4f4b] 2024-11-12T18:16:52,895 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/60183d34a1c24cd897d1c746939ff868 is 175, key is test_row_0/A:col10/1731435412311/Put/seqid=0 2024-11-12T18:16:52,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742505_1681 (size=22561) 2024-11-12T18:16:52,899 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/60183d34a1c24cd897d1c746939ff868 2024-11-12T18:16:52,904 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/8c44c37537e34eabbad43dbe90145d9b is 50, key is test_row_0/B:col10/1731435412311/Put/seqid=0 2024-11-12T18:16:52,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742506_1682 (size=9857) 2024-11-12T18:16:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T18:16:53,309 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/8c44c37537e34eabbad43dbe90145d9b 2024-11-12T18:16:53,326 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/68f413f9a2244b5bab0b99b95b570534 is 50, key is test_row_0/C:col10/1731435412311/Put/seqid=0 2024-11-12T18:16:53,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742507_1683 (size=9857) 2024-11-12T18:16:53,337 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/68f413f9a2244b5bab0b99b95b570534 2024-11-12T18:16:53,357 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/A/60183d34a1c24cd897d1c746939ff868 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/60183d34a1c24cd897d1c746939ff868 2024-11-12T18:16:53,368 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/60183d34a1c24cd897d1c746939ff868, entries=100, sequenceid=406, filesize=22.0 K 2024-11-12T18:16:53,369 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/B/8c44c37537e34eabbad43dbe90145d9b as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/8c44c37537e34eabbad43dbe90145d9b 2024-11-12T18:16:53,381 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/8c44c37537e34eabbad43dbe90145d9b, entries=100, sequenceid=406, filesize=9.6 K 2024-11-12T18:16:53,394 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/.tmp/C/68f413f9a2244b5bab0b99b95b570534 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/68f413f9a2244b5bab0b99b95b570534 2024-11-12T18:16:53,405 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/68f413f9a2244b5bab0b99b95b570534, entries=100, sequenceid=406, filesize=9.6 K 2024-11-12T18:16:53,407 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 928ms, sequenceid=406, compaction requested=false 2024-11-12T18:16:53,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84590914d15d40ffa62641fb319a7500, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/fbf15db17845446497726852af3afdd9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/398be3ed29854bed9168550f38149f7b, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4f1279a744cb48c1b1c3dbd86744d2b2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/9545b07512514dd5a4a1217dd50a86dc, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/dc332165c5c442f9a20bccc9b90108a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/cdb2f6cc3f94428992073f2b47b1e257, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c] to archive 2024-11-12T18:16:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:16:53,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/c532340d3ebc4e8e9ffb8025456bf3da 2024-11-12T18:16:53,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/04199cf9f3864f73ab89916e71592379 2024-11-12T18:16:53,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f4b1a73ca7eb4f26a00d2c7bf7ee451b 2024-11-12T18:16:53,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84590914d15d40ffa62641fb319a7500 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84590914d15d40ffa62641fb319a7500 2024-11-12T18:16:53,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4947390725be40dc94230499a8f5718d 2024-11-12T18:16:53,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/84bb2da5c38c43228d4c0c93c118c03c 2024-11-12T18:16:53,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/fbf15db17845446497726852af3afdd9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/fbf15db17845446497726852af3afdd9 2024-11-12T18:16:53,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/e908a4c6835f4f9ba59d84673a3fb8da 2024-11-12T18:16:53,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/f14fe48a6030413fa4af51a2f4459f9f 2024-11-12T18:16:53,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/3849ffd3cdae429ab14f26f361a71346 2024-11-12T18:16:53,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/398be3ed29854bed9168550f38149f7b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/398be3ed29854bed9168550f38149f7b 2024-11-12T18:16:53,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/91a5efc8b3b94585a7d9088f35e240e3 2024-11-12T18:16:53,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/2df9ef720deb43aba64946812e6f9ef4 2024-11-12T18:16:53,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0e196c4d91524a8b878b14e6bb6f7f98 2024-11-12T18:16:53,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4f1279a744cb48c1b1c3dbd86744d2b2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4f1279a744cb48c1b1c3dbd86744d2b2 2024-11-12T18:16:53,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/88381ee198074833b6e2795171962f11 2024-11-12T18:16:53,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/9545b07512514dd5a4a1217dd50a86dc to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/9545b07512514dd5a4a1217dd50a86dc 2024-11-12T18:16:53,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/259f5cf69e924b279cacd343d62a75ea 2024-11-12T18:16:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T18:16:53,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/bb54c33964a04227bb8dc2235615c71e 2024-11-12T18:16:53,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/dc332165c5c442f9a20bccc9b90108a4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/dc332165c5c442f9a20bccc9b90108a4 2024-11-12T18:16:53,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/b73dea59a50140479af31a123cbdb794 2024-11-12T18:16:53,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/45b1960258e544708abd3f4d146335f7 2024-11-12T18:16:53,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/0101e93e875f4dadab1ce0166454769c 2024-11-12T18:16:53,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/cdb2f6cc3f94428992073f2b47b1e257 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/cdb2f6cc3f94428992073f2b47b1e257 2024-11-12T18:16:53,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/4c278239415d4dbdb9a659af1f5f2a5d 2024-11-12T18:16:53,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/48c5bf2c18f24fc699fb2d62756efc4c 2024-11-12T18:16:53,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f29f7448a50041afa886b88a468c5adb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f01f612344314f5896dfceeff4197f10, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/30460af477664e95a826d61960f17ae4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/06abb00aa358425f8bf47d5b3e38cd53, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/17895edafd2749079a6d6af0e5db80d6, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/d8c164be06b04da3837c9cd9c01e0d22, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/4e3a80dd1f8b49ff8b1c645227265a48, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ddf8297ce9c34095a62325e81414dae4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e3783c94c56049bdb8f593d455299390, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/739f60ac0f0c4308b256f9758e058925, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2c8ae745de8b4b87b3933cf575b8ea2c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e4b8cd9ae6c54dbea197eaf0e91a26b9, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/97f4e19619cc40c2a2378049392f120d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e69f3831f4744d849f972adc36747fe3, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2fc7b4f0bcf34d9b8e654ee43753619f, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/0ed97b0cc3b647099a5bee30b019865a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/dde03ae8a3d84b7995d5845f2258b890, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/18a450df36b54cc9b5c3323844207694, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/19322806618f48d7aa7c2a3c4fb88051, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a9351fb0f5a04f81877a45ce8f5b65b2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/3ec967c5042b4a90af83585e6437c78c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ee91982cc12d4b8db86654f62f8a82a4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/c58cd248c41348cc95328f339c8646e8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/bf84ae42657645778c3c11eeadd9ca68, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a4ff2f1c09a94684a564133f82ac072a, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/1d0f923e93d344fbb3fc71a6cd777643] to archive 2024-11-12T18:16:53,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:16:53,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f29f7448a50041afa886b88a468c5adb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f29f7448a50041afa886b88a468c5adb 2024-11-12T18:16:53,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f01f612344314f5896dfceeff4197f10 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/f01f612344314f5896dfceeff4197f10 2024-11-12T18:16:53,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/30460af477664e95a826d61960f17ae4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/30460af477664e95a826d61960f17ae4 2024-11-12T18:16:53,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/06abb00aa358425f8bf47d5b3e38cd53 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/06abb00aa358425f8bf47d5b3e38cd53 2024-11-12T18:16:53,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/17895edafd2749079a6d6af0e5db80d6 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/17895edafd2749079a6d6af0e5db80d6 2024-11-12T18:16:53,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/d8c164be06b04da3837c9cd9c01e0d22 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/d8c164be06b04da3837c9cd9c01e0d22 2024-11-12T18:16:53,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/4e3a80dd1f8b49ff8b1c645227265a48 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/4e3a80dd1f8b49ff8b1c645227265a48 2024-11-12T18:16:53,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ddf8297ce9c34095a62325e81414dae4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ddf8297ce9c34095a62325e81414dae4 2024-11-12T18:16:53,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e3783c94c56049bdb8f593d455299390 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e3783c94c56049bdb8f593d455299390 2024-11-12T18:16:53,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/739f60ac0f0c4308b256f9758e058925 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/739f60ac0f0c4308b256f9758e058925 2024-11-12T18:16:53,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2c8ae745de8b4b87b3933cf575b8ea2c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2c8ae745de8b4b87b3933cf575b8ea2c 2024-11-12T18:16:53,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e4b8cd9ae6c54dbea197eaf0e91a26b9 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e4b8cd9ae6c54dbea197eaf0e91a26b9 2024-11-12T18:16:53,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/97f4e19619cc40c2a2378049392f120d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/97f4e19619cc40c2a2378049392f120d 2024-11-12T18:16:53,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e69f3831f4744d849f972adc36747fe3 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/e69f3831f4744d849f972adc36747fe3 2024-11-12T18:16:53,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2fc7b4f0bcf34d9b8e654ee43753619f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/2fc7b4f0bcf34d9b8e654ee43753619f 2024-11-12T18:16:53,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/0ed97b0cc3b647099a5bee30b019865a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/0ed97b0cc3b647099a5bee30b019865a 2024-11-12T18:16:53,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/dde03ae8a3d84b7995d5845f2258b890 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/dde03ae8a3d84b7995d5845f2258b890 2024-11-12T18:16:53,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/18a450df36b54cc9b5c3323844207694 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/18a450df36b54cc9b5c3323844207694 2024-11-12T18:16:53,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/19322806618f48d7aa7c2a3c4fb88051 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/19322806618f48d7aa7c2a3c4fb88051 2024-11-12T18:16:53,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a9351fb0f5a04f81877a45ce8f5b65b2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a9351fb0f5a04f81877a45ce8f5b65b2 2024-11-12T18:16:53,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/3ec967c5042b4a90af83585e6437c78c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/3ec967c5042b4a90af83585e6437c78c 2024-11-12T18:16:53,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ee91982cc12d4b8db86654f62f8a82a4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/ee91982cc12d4b8db86654f62f8a82a4 2024-11-12T18:16:53,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/c58cd248c41348cc95328f339c8646e8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/c58cd248c41348cc95328f339c8646e8 2024-11-12T18:16:53,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/bf84ae42657645778c3c11eeadd9ca68 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/bf84ae42657645778c3c11eeadd9ca68 2024-11-12T18:16:53,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a4ff2f1c09a94684a564133f82ac072a to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/a4ff2f1c09a94684a564133f82ac072a 2024-11-12T18:16:53,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/1d0f923e93d344fbb3fc71a6cd777643 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/1d0f923e93d344fbb3fc71a6cd777643 2024-11-12T18:16:53,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/77a5f454a9df4cb8af5dfffd8d62e611, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e6347e40fc19490fac5b87344228ea45, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/cc3c148735994a92b77b863634d9d3df, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69836a772ced4302b122e646a673b7e1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ff1728ab47ec4baf9f81886505a57a18, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69c10980877b4c189ea9b7ad1007aaef, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f643dd18b5ae442abd115d7bea1caebf, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36b589f16928490ca57b08b7abbfd1bb, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/828b6eedceba437a8ca963da08b090a7, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b27e3316fce948529d3251b38c8d85ae, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/be2051941af64cf88fef4df3d2592063, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36d2c9db34a14a68aeccfcdee0279bc2, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/615147057b1440d88e169c18548f5873, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f87eae5c409c48b3bfcc33c30f034a5d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b41eed0c841a43c4bbea6caa9380059c, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/54623c654c824fd0abef0765c1a7b311, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/a6565b0e07eb463eb196dbe6576b5f99, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/fd0990c4f3044d63845e741bc434dbb8, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ea4f061a9c4e43808dca2237eaad2896, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/6a8446568e8742078c72f090fb88a07d, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/5e1fcbe2ca614a06add39db9ed7e96f1, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3c5a8466c41d4d9cafa83d9fac7ba4f4, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3d27e5004e3042e089405d2532855c94, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8cc7320a7874d65b68bf6a3c9f8a011, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/dbdfc272c07f400395f1081c8ea6bd2f] to archive 2024-11-12T18:16:53,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T18:16:53,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/77a5f454a9df4cb8af5dfffd8d62e611 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/77a5f454a9df4cb8af5dfffd8d62e611 2024-11-12T18:16:53,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e6347e40fc19490fac5b87344228ea45 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e6347e40fc19490fac5b87344228ea45 2024-11-12T18:16:53,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/cc3c148735994a92b77b863634d9d3df to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/cc3c148735994a92b77b863634d9d3df 2024-11-12T18:16:53,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69836a772ced4302b122e646a673b7e1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69836a772ced4302b122e646a673b7e1 2024-11-12T18:16:53,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ff1728ab47ec4baf9f81886505a57a18 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ff1728ab47ec4baf9f81886505a57a18 2024-11-12T18:16:53,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69c10980877b4c189ea9b7ad1007aaef to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/69c10980877b4c189ea9b7ad1007aaef 2024-11-12T18:16:53,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f643dd18b5ae442abd115d7bea1caebf to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f643dd18b5ae442abd115d7bea1caebf 2024-11-12T18:16:53,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36b589f16928490ca57b08b7abbfd1bb to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36b589f16928490ca57b08b7abbfd1bb 2024-11-12T18:16:53,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/828b6eedceba437a8ca963da08b090a7 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/828b6eedceba437a8ca963da08b090a7 2024-11-12T18:16:53,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8c03468bb1e4dcbb0c3e7f9f0d22fbe 2024-11-12T18:16:53,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b27e3316fce948529d3251b38c8d85ae to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b27e3316fce948529d3251b38c8d85ae 2024-11-12T18:16:53,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/be2051941af64cf88fef4df3d2592063 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/be2051941af64cf88fef4df3d2592063 2024-11-12T18:16:53,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36d2c9db34a14a68aeccfcdee0279bc2 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/36d2c9db34a14a68aeccfcdee0279bc2 2024-11-12T18:16:53,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/615147057b1440d88e169c18548f5873 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/615147057b1440d88e169c18548f5873 2024-11-12T18:16:53,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f87eae5c409c48b3bfcc33c30f034a5d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/f87eae5c409c48b3bfcc33c30f034a5d 2024-11-12T18:16:53,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b41eed0c841a43c4bbea6caa9380059c to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/b41eed0c841a43c4bbea6caa9380059c 2024-11-12T18:16:53,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/54623c654c824fd0abef0765c1a7b311 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/54623c654c824fd0abef0765c1a7b311 2024-11-12T18:16:53,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/a6565b0e07eb463eb196dbe6576b5f99 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/a6565b0e07eb463eb196dbe6576b5f99 2024-11-12T18:16:53,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/fd0990c4f3044d63845e741bc434dbb8 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/fd0990c4f3044d63845e741bc434dbb8 2024-11-12T18:16:53,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ea4f061a9c4e43808dca2237eaad2896 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/ea4f061a9c4e43808dca2237eaad2896 2024-11-12T18:16:53,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/6a8446568e8742078c72f090fb88a07d to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/6a8446568e8742078c72f090fb88a07d 2024-11-12T18:16:53,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/5e1fcbe2ca614a06add39db9ed7e96f1 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/5e1fcbe2ca614a06add39db9ed7e96f1 2024-11-12T18:16:53,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3c5a8466c41d4d9cafa83d9fac7ba4f4 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3c5a8466c41d4d9cafa83d9fac7ba4f4 2024-11-12T18:16:53,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3d27e5004e3042e089405d2532855c94 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/3d27e5004e3042e089405d2532855c94 2024-11-12T18:16:53,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8cc7320a7874d65b68bf6a3c9f8a011 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/e8cc7320a7874d65b68bf6a3c9f8a011 2024-11-12T18:16:53,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/dbdfc272c07f400395f1081c8ea6bd2f to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/dbdfc272c07f400395f1081c8ea6bd2f 2024-11-12T18:16:53,611 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/recovered.edits/409.seqid, newMaxSeqId=409, maxSeqId=4 2024-11-12T18:16:53,612 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. 2024-11-12T18:16:53,612 DEBUG [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for ffab90d6ee04bb318f8ea8ffd27a4f4b: Waiting for close lock at 1731435412478Running coprocessor pre-close hooks at 1731435412478Disabling compacts and flushes for region at 1731435412478Disabling writes for close at 1731435412479 (+1 ms)Obtaining lock to block concurrent updates at 1731435412479Preparing flush snapshotting stores in ffab90d6ee04bb318f8ea8ffd27a4f4b at 1731435412479Finished memstore snapshotting TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731435412479Flushing stores of TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b. at 1731435412480 (+1 ms)Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/A: creating writer at 1731435412480Flushing mob file ffab90d6ee04bb318f8ea8ffd27a4f4b/A: appending metadata at 1731435412484 (+4 ms)Flushing mob file ffab90d6ee04bb318f8ea8ffd27a4f4b/A: closing flushed file at 1731435412484Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/A: appending metadata at 1731435412895 (+411 ms)Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/A: closing flushed file at 1731435412895Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/B: creating writer at 1731435412902 (+7 ms)Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/B: appending metadata at 1731435412904 (+2 ms)Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/B: closing flushed file at 1731435412904Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/C: creating writer at 1731435413323 (+419 ms)Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/C: appending metadata at 1731435413326 (+3 ms)Flushing ffab90d6ee04bb318f8ea8ffd27a4f4b/C: closing flushed file at 1731435413326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@214432c6: reopening flushed file at 1731435413347 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33afb051: reopening flushed file at 1731435413368 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a5d4c59: reopening flushed file at 1731435413381 (+13 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ffab90d6ee04bb318f8ea8ffd27a4f4b in 928ms, sequenceid=406, compaction requested=false at 1731435413407 (+26 ms)Writing region close event to WAL at 1731435413608 (+201 ms)Running coprocessor post-close hooks at 1731435413612 (+4 ms)Closed at 1731435413612 2024-11-12T18:16:53,614 INFO [RS_CLOSE_REGION-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:53,614 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=ffab90d6ee04bb318f8ea8ffd27a4f4b, regionState=CLOSED 2024-11-12T18:16:53,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 because future has completed 2024-11-12T18:16:53,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-12T18:16:53,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure ffab90d6ee04bb318f8ea8ffd27a4f4b, server=9911683f163c,43757,1731435233300 in 1.2910 sec 2024-11-12T18:16:53,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-12T18:16:53,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ffab90d6ee04bb318f8ea8ffd27a4f4b, UNASSIGN in 1.2960 sec 2024-11-12T18:16:53,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-12T18:16:53,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3050 sec 2024-11-12T18:16:53,631 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731435413630"}]},"ts":"1731435413630"} 2024-11-12T18:16:53,632 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T18:16:53,633 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T18:16:53,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3180 sec 2024-11-12T18:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-12T18:16:54,471 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:54,471 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T18:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:54,473 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:54,474 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-12T18:16:54,476 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,479 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C, FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/recovered.edits] 2024-11-12T18:16:54,481 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/300ad5cf230b4d63bc1e9a41c7571492 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/300ad5cf230b4d63bc1e9a41c7571492 2024-11-12T18:16:54,482 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/60183d34a1c24cd897d1c746939ff868 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/A/60183d34a1c24cd897d1c746939ff868 2024-11-12T18:16:54,485 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/8c44c37537e34eabbad43dbe90145d9b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/8c44c37537e34eabbad43dbe90145d9b 2024-11-12T18:16:54,486 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/adc217607eea4949833df47fa2a55256 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/B/adc217607eea4949833df47fa2a55256 2024-11-12T18:16:54,494 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/500abbc978cd4c109e9f162be2cbbe59 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/500abbc978cd4c109e9f162be2cbbe59 2024-11-12T18:16:54,495 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/68f413f9a2244b5bab0b99b95b570534 to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/C/68f413f9a2244b5bab0b99b95b570534 2024-11-12T18:16:54,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/recovered.edits/409.seqid to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b/recovered.edits/409.seqid 2024-11-12T18:16:54,498 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/default/TestAcidGuarantees/ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,500 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T18:16:54,501 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T18:16:54,502 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T18:16:54,504 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112145b0e10519a4f00b6d7b7c17e966843_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112145b0e10519a4f00b6d7b7c17e966843_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,505 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122409f25fd7f242968b0110521e821ec4_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122409f25fd7f242968b0110521e821ec4_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,506 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122695cc3d32144a9a80dd89f917452f13_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122695cc3d32144a9a80dd89f917452f13_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,507 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112428a1d897757488387457f570e5aef66_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112428a1d897757488387457f570e5aef66_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,509 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124b5d80a314c7400d8f11cf2dfe89fa0f_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124b5d80a314c7400d8f11cf2dfe89fa0f_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,510 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111251b0ba1318d841bd9e6fef18e967e487_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111251b0ba1318d841bd9e6fef18e967e487_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,511 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125376665f1986403e8ca792dfc338b327_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125376665f1986403e8ca792dfc338b327_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,512 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260e7fd8d26484c079fabcd6ab4308dc7_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260e7fd8d26484c079fabcd6ab4308dc7_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,513 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112624f8765d8a74f0aac081341682e3f94_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112624f8765d8a74f0aac081341682e3f94_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126307607727244766858f2bfccec1af2e_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126307607727244766858f2bfccec1af2e_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,516 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111294c029dbe3824b6c93e9b0a5726eb6e5_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111294c029dbe3824b6c93e9b0a5726eb6e5_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,517 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129aada533b1b4457ab4805de551f50ec7_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129aada533b1b4457ab4805de551f50ec7_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,518 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b0eac5ba3816467bb1c1a6516cb1c677_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b0eac5ba3816467bb1c1a6516cb1c677_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,519 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b35de83e571c4e76a40a9eb0ebf75b43_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b35de83e571c4e76a40a9eb0ebf75b43_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,520 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c2b7968d7e084474a2bf7f2c89050e6c_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c2b7968d7e084474a2bf7f2c89050e6c_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,521 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c9462ff1978549a6a8a2125abed11440_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c9462ff1978549a6a8a2125abed11440_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,522 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d85c2fc6868f4fb39cf3538ea08b5e13_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d85c2fc6868f4fb39cf3538ea08b5e13_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,524 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e1373a3e0d15407d97944ef0dd7126c9_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e1373a3e0d15407d97944ef0dd7126c9_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,528 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f386b2a3c4c64c6bb01c165ba3d82d32_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f386b2a3c4c64c6bb01c165ba3d82d32_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fd18dcc04f94422fa675eaf4c39eba1d_ffab90d6ee04bb318f8ea8ffd27a4f4b to hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fd18dcc04f94422fa675eaf4c39eba1d_ffab90d6ee04bb318f8ea8ffd27a4f4b 2024-11-12T18:16:54,532 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T18:16:54,535 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:54,537 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T18:16:54,539 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T18:16:54,541 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:54,541 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T18:16:54,541 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731435414541"}]},"ts":"9223372036854775807"} 2024-11-12T18:16:54,543 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T18:16:54,543 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ffab90d6ee04bb318f8ea8ffd27a4f4b, NAME => 'TestAcidGuarantees,,1731435378462.ffab90d6ee04bb318f8ea8ffd27a4f4b.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T18:16:54,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T18:16:54,543 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731435414543"}]},"ts":"9223372036854775807"} 2024-11-12T18:16:54,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T18:16:54,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T18:16:54,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-11-12T18:16:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-12T18:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T18:16:54,581 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T18:16:54,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 252), OpenFileDescriptor=445 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=685 (was 644) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=534 (was 476) - AvailableMemoryMB LEAK? - 2024-11-12T18:16:54,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T18:16:54,593 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:16:54,593 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:16:54,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:54,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:54,594 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T18:16:54,594 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-12T18:16:54,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T18:16:54,594 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9911683f163c,39807,1731435232358 2024-11-12T18:16:54,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T18:16:54,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:16:54,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T18:16:54,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:16:54,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:16:54,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:16:54,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T18:16:54,599 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T18:16:54,599 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:16:54,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:54,600 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9911683f163c,43757,1731435233300' ***** 2024-11-12T18:16:54,600 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T18:16:54,600 INFO [RS:0;9911683f163c:43757 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T18:16:54,601 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(959): stopping server 9911683f163c,43757,1731435233300 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9911683f163c:43757. 2024-11-12T18:16:54,601 DEBUG [RS:0;9911683f163c:43757 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T18:16:54,601 DEBUG [RS:0;9911683f163c:43757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T18:16:54,601 INFO [RS:0;9911683f163c:43757 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T18:16:54,602 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T18:16:54,602 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T18:16:54,602 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T18:16:54,603 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T18:16:54,603 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T18:16:54,603 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T18:16:54,603 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T18:16:54,603 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T18:16:54,603 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T18:16:54,603 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T18:16:54,632 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/info/f7a99279183d47c2a39c7511f8889b28 is 95, key is TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577./info:/1731435260527/DeleteFamily/seqid=0 2024-11-12T18:16:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742508_1684 (size=6514) 2024-11-12T18:16:54,637 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/info/f7a99279183d47c2a39c7511f8889b28 2024-11-12T18:16:54,661 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/ns/f3cb05ce9a5e4031bf1a7e486bb473b5 is 93, key is TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577./ns:/1731435260427/DeleteFamily/seqid=0 2024-11-12T18:16:54,663 INFO [regionserver/9911683f163c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:16:54,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742509_1685 (size=5906) 2024-11-12T18:16:54,666 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/ns/f3cb05ce9a5e4031bf1a7e486bb473b5 2024-11-12T18:16:54,697 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/rep_barrier/d91f4b8b073d4bac9a7d8d8bcf5c3348 is 102, key is TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577./rep_barrier:/1731435260427/DeleteFamily/seqid=0 2024-11-12T18:16:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742510_1686 (size=6025) 2024-11-12T18:16:54,803 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T18:16:55,003 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T18:16:55,017 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T18:16:55,017 INFO [regionserver/9911683f163c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T18:16:55,105 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/rep_barrier/d91f4b8b073d4bac9a7d8d8bcf5c3348 2024-11-12T18:16:55,133 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/table/73d3e653b4e842d3a78c67d480d081e4 is 96, key is TestAcidGuarantees,,1731435237428.64a54f5030b48a2f7caee1e78dcb2577./table:/1731435260427/DeleteFamily/seqid=0 2024-11-12T18:16:55,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742511_1687 (size=5936) 2024-11-12T18:16:55,203 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T18:16:55,404 DEBUG [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T18:16:55,537 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/table/73d3e653b4e842d3a78c67d480d081e4 2024-11-12T18:16:55,541 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/info/f7a99279183d47c2a39c7511f8889b28 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/info/f7a99279183d47c2a39c7511f8889b28 2024-11-12T18:16:55,544 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/info/f7a99279183d47c2a39c7511f8889b28, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T18:16:55,545 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/ns/f3cb05ce9a5e4031bf1a7e486bb473b5 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/ns/f3cb05ce9a5e4031bf1a7e486bb473b5 2024-11-12T18:16:55,549 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/ns/f3cb05ce9a5e4031bf1a7e486bb473b5, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T18:16:55,549 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/rep_barrier/d91f4b8b073d4bac9a7d8d8bcf5c3348 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/rep_barrier/d91f4b8b073d4bac9a7d8d8bcf5c3348 2024-11-12T18:16:55,553 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/rep_barrier/d91f4b8b073d4bac9a7d8d8bcf5c3348, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T18:16:55,553 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/.tmp/table/73d3e653b4e842d3a78c67d480d081e4 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/table/73d3e653b4e842d3a78c67d480d081e4 2024-11-12T18:16:55,556 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/table/73d3e653b4e842d3a78c67d480d081e4, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T18:16:55,557 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 954ms, sequenceid=90, compaction requested=false 2024-11-12T18:16:55,567 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T18:16:55,568 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T18:16:55,568 INFO [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T18:16:55,568 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731435414602Running coprocessor pre-close hooks at 1731435414603 (+1 ms)Disabling compacts and flushes for region at 1731435414603Disabling writes for close at 1731435414603Obtaining lock to block concurrent updates at 1731435414603Preparing flush snapshotting stores in 1588230740 at 1731435414603Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731435414603Flushing stores of hbase:meta,,1.1588230740 at 1731435414604 (+1 ms)Flushing 1588230740/info: creating writer at 1731435414604Flushing 1588230740/info: appending metadata at 1731435414631 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731435414631Flushing 1588230740/ns: creating writer at 1731435414642 (+11 ms)Flushing 1588230740/ns: appending metadata at 1731435414661 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731435414661Flushing 1588230740/rep_barrier: creating writer at 1731435414673 (+12 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731435414697 (+24 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731435414697Flushing 1588230740/table: creating writer at 1731435415110 (+413 ms)Flushing 1588230740/table: appending metadata at 1731435415132 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731435415132Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@267c6da2: reopening flushed file at 1731435415540 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fa85e19: reopening flushed file at 1731435415545 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e22293f: reopening flushed file at 1731435415549 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d45d970: reopening flushed file at 1731435415553 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 954ms, sequenceid=90, compaction requested=false at 1731435415557 (+4 ms)Writing region close event to WAL at 1731435415565 (+8 ms)Running coprocessor post-close hooks at 1731435415568 (+3 ms)Closed at 1731435415568 2024-11-12T18:16:55,568 DEBUG [RS_CLOSE_META-regionserver/9911683f163c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T18:16:55,604 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(976): stopping server 9911683f163c,43757,1731435233300; all regions closed. 2024-11-12T18:16:55,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741834_1010 (size=25344) 2024-11-12T18:16:55,610 DEBUG [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/oldWALs 2024-11-12T18:16:55,611 INFO [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9911683f163c%2C43757%2C1731435233300.meta:.meta(num 1731435236750) 2024-11-12T18:16:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741833_1009 (size=22287428) 2024-11-12T18:16:55,614 DEBUG [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/oldWALs 2024-11-12T18:16:55,614 INFO [RS:0;9911683f163c:43757 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9911683f163c%2C43757%2C1731435233300:(num 1731435236165) 2024-11-12T18:16:55,614 DEBUG [RS:0;9911683f163c:43757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T18:16:55,614 INFO [RS:0;9911683f163c:43757 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T18:16:55,614 INFO [RS:0;9911683f163c:43757 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T18:16:55,614 INFO [RS:0;9911683f163c:43757 {}] hbase.ChoreService(370): Chore service for: regionserver/9911683f163c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T18:16:55,614 INFO [RS:0;9911683f163c:43757 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T18:16:55,614 INFO [regionserver/9911683f163c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T18:16:55,615 INFO [RS:0;9911683f163c:43757 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43757 2024-11-12T18:16:55,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9911683f163c,43757,1731435233300 2024-11-12T18:16:55,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T18:16:55,619 INFO [RS:0;9911683f163c:43757 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T18:16:55,621 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9911683f163c,43757,1731435233300] 2024-11-12T18:16:55,623 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9911683f163c,43757,1731435233300 already deleted, retry=false 2024-11-12T18:16:55,623 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9911683f163c,43757,1731435233300 expired; onlineServers=0 2024-11-12T18:16:55,623 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9911683f163c,39807,1731435232358' ***** 2024-11-12T18:16:55,623 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T18:16:55,623 INFO [M:0;9911683f163c:39807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T18:16:55,623 INFO [M:0;9911683f163c:39807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T18:16:55,623 DEBUG [M:0;9911683f163c:39807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T18:16:55,623 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T18:16:55,623 DEBUG [M:0;9911683f163c:39807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T18:16:55,623 DEBUG [master/9911683f163c:0:becomeActiveMaster-HFileCleaner.small.0-1731435235569 {}] cleaner.HFileCleaner(306): Exit Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.small.0-1731435235569,5,FailOnTimeoutGroup] 2024-11-12T18:16:55,623 DEBUG [master/9911683f163c:0:becomeActiveMaster-HFileCleaner.large.0-1731435235567 {}] cleaner.HFileCleaner(306): Exit Thread[master/9911683f163c:0:becomeActiveMaster-HFileCleaner.large.0-1731435235567,5,FailOnTimeoutGroup] 2024-11-12T18:16:55,624 INFO [M:0;9911683f163c:39807 {}] hbase.ChoreService(370): Chore service for: master/9911683f163c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T18:16:55,624 INFO [M:0;9911683f163c:39807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T18:16:55,624 DEBUG [M:0;9911683f163c:39807 {}] master.HMaster(1795): Stopping service threads 2024-11-12T18:16:55,624 INFO [M:0;9911683f163c:39807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T18:16:55,624 INFO [M:0;9911683f163c:39807 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T18:16:55,624 ERROR [M:0;9911683f163c:39807 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T18:16:55,625 INFO [M:0;9911683f163c:39807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T18:16:55,625 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T18:16:55,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T18:16:55,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T18:16:55,625 DEBUG [M:0;9911683f163c:39807 {}] zookeeper.ZKUtil(347): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T18:16:55,625 WARN [M:0;9911683f163c:39807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T18:16:55,626 INFO [M:0;9911683f163c:39807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/.lastflushedseqids 2024-11-12T18:16:55,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742512_1688 (size=72) 2024-11-12T18:16:55,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:16:55,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43757-0x100353303020001, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:16:55,722 INFO [RS:0;9911683f163c:43757 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T18:16:55,722 INFO [RS:0;9911683f163c:43757 {}] regionserver.HRegionServer(1031): Exiting; stopping=9911683f163c,43757,1731435233300; zookeeper connection closed. 2024-11-12T18:16:55,723 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ea9fb3a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ea9fb3a 2024-11-12T18:16:55,723 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T18:16:56,036 INFO [M:0;9911683f163c:39807 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T18:16:56,036 INFO [M:0;9911683f163c:39807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T18:16:56,036 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T18:16:56,037 INFO [M:0;9911683f163c:39807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:16:56,037 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:16:56,037 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T18:16:56,037 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:16:56,037 INFO [M:0;9911683f163c:39807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=777.11 KB heapSize=951.98 KB 2024-11-12T18:16:56,052 DEBUG [M:0;9911683f163c:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f52a0b66b3446fc9dcbe8aeb9e53ed3 is 82, key is hbase:meta,,1/info:regioninfo/1731435236879/Put/seqid=0 2024-11-12T18:16:56,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742513_1689 (size=5672) 2024-11-12T18:16:56,456 INFO [M:0;9911683f163c:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2156 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f52a0b66b3446fc9dcbe8aeb9e53ed3 2024-11-12T18:16:56,476 DEBUG [M:0;9911683f163c:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/825fef722eae419fbb31069366e76952 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731435381553/Put/seqid=0 2024-11-12T18:16:56,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742514_1690 (size=45696) 2024-11-12T18:16:56,880 INFO [M:0;9911683f163c:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=776.55 KB at sequenceid=2156 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/825fef722eae419fbb31069366e76952 2024-11-12T18:16:56,883 INFO [M:0;9911683f163c:39807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 825fef722eae419fbb31069366e76952 2024-11-12T18:16:56,899 DEBUG [M:0;9911683f163c:39807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a84e6867d4f24928aca3372a25a02fa2 is 69, key is 9911683f163c,43757,1731435233300/rs:state/1731435235712/Put/seqid=0 2024-11-12T18:16:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073742515_1691 (size=5156) 2024-11-12T18:16:57,303 INFO [M:0;9911683f163c:39807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2156 (bloomFilter=true), to=hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a84e6867d4f24928aca3372a25a02fa2 2024-11-12T18:16:57,306 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f52a0b66b3446fc9dcbe8aeb9e53ed3 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9f52a0b66b3446fc9dcbe8aeb9e53ed3 2024-11-12T18:16:57,309 INFO [M:0;9911683f163c:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9f52a0b66b3446fc9dcbe8aeb9e53ed3, entries=8, sequenceid=2156, filesize=5.5 K 2024-11-12T18:16:57,310 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/825fef722eae419fbb31069366e76952 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/825fef722eae419fbb31069366e76952 2024-11-12T18:16:57,312 INFO [M:0;9911683f163c:39807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 825fef722eae419fbb31069366e76952 2024-11-12T18:16:57,312 INFO [M:0;9911683f163c:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/825fef722eae419fbb31069366e76952, entries=172, sequenceid=2156, filesize=44.6 K 2024-11-12T18:16:57,313 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a84e6867d4f24928aca3372a25a02fa2 as hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a84e6867d4f24928aca3372a25a02fa2 2024-11-12T18:16:57,315 INFO [M:0;9911683f163c:39807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41781/user/jenkins/test-data/5c1dcec2-5ee9-ba41-2b44-a1e96ec8c1b7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a84e6867d4f24928aca3372a25a02fa2, entries=1, sequenceid=2156, filesize=5.0 K 2024-11-12T18:16:57,316 INFO [M:0;9911683f163c:39807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~777.11 KB/795759, heapSize ~951.69 KB/974528, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2156, compaction requested=false 2024-11-12T18:16:57,317 INFO [M:0;9911683f163c:39807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T18:16:57,317 DEBUG [M:0;9911683f163c:39807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731435416036Disabling compacts and flushes for region at 1731435416036Disabling writes for close at 1731435416037 (+1 ms)Obtaining lock to block concurrent updates at 1731435416037Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731435416037Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=795759, getHeapSize=974768, getOffHeapSize=0, getCellsCount=2353 at 1731435416037Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731435416037Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731435416038 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731435416052 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731435416052Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731435416459 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731435416475 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731435416475Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731435416883 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731435416898 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731435416898Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20dc81ff: reopening flushed file at 1731435417306 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bb75b84: reopening flushed file at 1731435417309 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689d518: reopening flushed file at 1731435417312 (+3 ms)Finished flush of dataSize ~777.11 KB/795759, heapSize ~951.69 KB/974528, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2156, compaction requested=false at 1731435417316 (+4 ms)Writing region close event to WAL at 1731435417317 (+1 ms)Closed at 1731435417317 2024-11-12T18:16:57,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39361 is added to blk_1073741830_1006 (size=937106) 2024-11-12T18:16:57,320 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T18:16:57,320 INFO [M:0;9911683f163c:39807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T18:16:57,320 INFO [M:0;9911683f163c:39807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39807 2024-11-12T18:16:57,321 INFO [M:0;9911683f163c:39807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T18:16:57,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:16:57,423 INFO [M:0;9911683f163c:39807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T18:16:57,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39807-0x100353303020000, quorum=127.0.0.1:59986, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T18:16:57,427 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T18:16:57,430 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T18:16:57,430 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T18:16:57,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T18:16:57,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/hadoop.log.dir/,STOPPED} 2024-11-12T18:16:57,434 WARN [BP-2004937493-172.17.0.3-1731435228362 heartbeating to localhost/127.0.0.1:41781 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T18:16:57,434 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T18:16:57,434 WARN [BP-2004937493-172.17.0.3-1731435228362 heartbeating to localhost/127.0.0.1:41781 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2004937493-172.17.0.3-1731435228362 (Datanode Uuid f7670328-f5c7-49f9-8286-33a4d3629853) service to localhost/127.0.0.1:41781 2024-11-12T18:16:57,434 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T18:16:57,437 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/data/data1/current/BP-2004937493-172.17.0.3-1731435228362 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T18:16:57,437 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/cluster_a891cfae-d6c8-bb1e-08ac-58df3f756317/data/data2/current/BP-2004937493-172.17.0.3-1731435228362 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T18:16:57,438 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T18:16:57,446 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T18:16:57,447 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T18:16:57,447 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T18:16:57,447 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T18:16:57,447 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a1187e36-8058-7ba8-9de6-b4643506f23c/hadoop.log.dir/,STOPPED} 2024-11-12T18:16:57,465 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T18:16:57,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down